From 34beb7f01e7682c37abef9cd2d8dd9789e5e4f6a Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 23 Sep 2026 20:30:20 +0800 Subject: [PATCH] Add bounded Omni llama.cpp Spark text stage for Windows CPU and Radeon --- .../probe_omni_llamacpp_entrypoint.py | 101 ++ .../experiments/probe_omni_llamacpp_spark.py | 192 ++++ .../results/e2e_expansion_20260923/README.md | 4 +- .../evidence/spark_omni_llamacpp/README.md | 24 + ..._cpu_public_realconfig_20260923.driver.log | 51 + ...amacpp_cpu_public_realconfig_20260923.json | 25 + ...lamacpp_cpu_public_realconfig_20260923.log | 212 +++++ ...lamacpp_cpu_tokenbound_20260923.driver.log | 136 +++ ...omni_llamacpp_cpu_tokenbound_20260923.json | 745 +++++++++++++++ .../omni_llamacpp_cpu_tokenbound_20260923.log | 893 ++++++++++++++++++ ...cpp_radeon_final_smoke_20260923.driver.log | 171 ++++ ..._llamacpp_radeon_final_smoke_20260923.json | 199 ++++ ...i_llamacpp_radeon_final_smoke_20260923.log | 299 ++++++ ...deon_public_realconfig_20260923.driver.log | 51 + ...cpp_radeon_public_realconfig_20260923.json | 25 + ...acpp_radeon_public_realconfig_20260923.log | 215 +++++ ...acpp_radeon_tokenbound_20260923.driver.log | 171 ++++ ...i_llamacpp_radeon_tokenbound_20260923.json | 780 +++++++++++++++ ...ni_llamacpp_radeon_tokenbound_20260923.log | 879 +++++++++++++++++ ..._llamacpp_wrong_device_20260923.driver.log | 101 ++ .../omni_llamacpp_wrong_device_20260923.json | 27 + .../omni_llamacpp_wrong_device_20260923.log | 191 ++++ .../spark_radeon890m_llamacpp/README.md | 4 +- .../evidence/summary/README.md | 12 +- docs/design/omni_edge_backends.md | 14 +- docs/design/omni_edge_e2e_roadmap.md | 2 + vllm_omni/config/pipeline_registry.py | 6 +- vllm_omni/engine/backends/__init__.py | 14 +- vllm_omni/engine/backends/llamacpp.py | 378 ++++++++ .../models/spark2_5/pipeline.py | 17 + 30 files changed, 5920 insertions(+), 19 deletions(-) create mode 100644 benchmarks/edge_harness/experiments/probe_omni_llamacpp_entrypoint.py create mode 100644 benchmarks/edge_harness/experiments/probe_omni_llamacpp_spark.py create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_public_realconfig_20260923.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_public_realconfig_20260923.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_public_realconfig_20260923.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_tokenbound_20260923.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_tokenbound_20260923.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_tokenbound_20260923.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_final_smoke_20260923.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_final_smoke_20260923.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_final_smoke_20260923.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_public_realconfig_20260923.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_public_realconfig_20260923.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_public_realconfig_20260923.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_tokenbound_20260923.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_tokenbound_20260923.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_tokenbound_20260923.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_wrong_device_20260923.driver.log create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_wrong_device_20260923.json create mode 100644 benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_wrong_device_20260923.log create mode 100644 vllm_omni/engine/backends/llamacpp.py diff --git a/benchmarks/edge_harness/experiments/probe_omni_llamacpp_entrypoint.py b/benchmarks/edge_harness/experiments/probe_omni_llamacpp_entrypoint.py new file mode 100644 index 000000000..2508cb684 --- /dev/null +++ b/benchmarks/edge_harness/experiments/probe_omni_llamacpp_entrypoint.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Check a pinned Spark GGUF through the public AsyncOmni complete-request API.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import tempfile +import time +from pathlib import Path + +import yaml + + +async def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + for name in ("model_file", "server_bin", "model_sha256", "server_sha256", "device", "server_log", "output_report"): + parser.add_argument("--" + name.replace("_", "-"), required=True) + parser.add_argument("--expected-device-name") + parser.add_argument("--model-config-dir", help="Existing Spark HF config directory for public resolution") + args = parser.parse_args() + + from vllm_omni.model_executor.models.spark2_5.pipeline import SPARK2_5_GGUF_TEXT_PIPELINE + from vllm_omni.entrypoints.async_omni import AsyncOmni + + pipeline = SPARK2_5_GGUF_TEXT_PIPELINE + backend = { + "name": "external.llamacpp.text.v1", + "model_file": args.model_file, + "model_sha256": args.model_sha256, + "server_bin": args.server_bin, + "server_sha256": args.server_sha256, + "log_file": args.server_log, + "device": args.device, + "expected_device_name": args.expected_device_name, + "context_tokens": 4096, + "max_new_tokens": 96, + "max_io_bytes": 1 << 20, + "memory_overhead_bytes": 2 << 30, + } + report = {"device": args.device, "entrypoint": "AsyncOmni.generate"} + engine = None + try: + with tempfile.TemporaryDirectory(prefix="omni-spark-gguf-") as temporary: + directory = Path(temporary) + if args.model_config_dir: + model_config_dir = Path(args.model_config_dir).resolve(strict=True) + else: + (directory / "config.json").write_text('{"model_type":"bert"}', encoding="utf-8") + model_config_dir = directory + deployment = directory / "deploy.yaml" + deployment.write_text(yaml.safe_dump({ + "pipeline": pipeline.model_type, + "async_chunk": False, + "stages": [{ + "stage_id": 0, + "backend": backend, + "resource_budget": { + "capacities": {"host_ram": 12 << 30}, + "demands": {"host_ram": 4 << 30}, + }, + }], + }), encoding="utf-8") + started = time.perf_counter() + engine = AsyncOmni( + model=str(model_config_dir), deploy_config=str(deployment), + stage_init_timeout=120, init_timeout=180, + ) + report["startup_s"] = time.perf_counter() - started + outputs = [] + started = time.perf_counter() + async for output in engine.generate( + {"text": "What is the capital of France? Answer in one word.", "max_tokens": 96}, + request_id="public-spark-1", + ): + outputs.append({ + "request_id": output.request_id, + "text": output.outputs[0].text if output.outputs else None, + "error": output.error, + "stage_event": output.custom_output.get("stage_event"), + }) + report["request_wall_s"] = time.perf_counter() - started + report["outputs"] = outputs + assert len(outputs) == 1 and outputs[0]["request_id"] == "public-spark-1" + assert not outputs[0]["error"] and outputs[0]["text"].strip() == "Paris" + report["status"] = "passed" + except BaseException as exc: + report["status"] = "failed" + report["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + if engine is not None: + engine.shutdown() + Path(args.output_report).parent.mkdir(parents=True, exist_ok=True) + Path(args.output_report).write_text(json.dumps(report, indent=2), encoding="utf-8") + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/benchmarks/edge_harness/experiments/probe_omni_llamacpp_spark.py b/benchmarks/edge_harness/experiments/probe_omni_llamacpp_spark.py new file mode 100644 index 000000000..506a86ed9 --- /dev/null +++ b/benchmarks/edge_harness/experiments/probe_omni_llamacpp_spark.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Exercise a pinned Spark GGUF through Omni StageRuntime and StagePool. + +Runs complete, serial text requests. This does not claim incremental streaming, +quality beyond the named inputs, or device-memory peak attribution. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import math +import time +from pathlib import Path +from types import SimpleNamespace + + +def _rank(values: list[float], fraction: float) -> float: + ordered = sorted(values) + return ordered[math.ceil(fraction * len(ordered)) - 1] + + +async def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-file", type=Path, required=True) + parser.add_argument("--server-bin", type=Path, required=True) + parser.add_argument("--server-sha256", required=True) + parser.add_argument("--model-sha256", required=True) + parser.add_argument("--device", required=True) + parser.add_argument("--expected-device-name") + parser.add_argument("--ggml-vk-visible-devices") + parser.add_argument("--server-log", type=Path, required=True) + parser.add_argument("--output-report", type=Path, required=True) + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--repeats", type=int, default=20) + parser.add_argument("--capacity-gib", type=int, default=12) + parser.add_argument("--reserve-gib", type=int, default=4) + parser.add_argument("--abort-check", action="store_true") + parser.add_argument("--context-check", action="store_true") + args = parser.parse_args() + if args.warmups < 0 or args.repeats <= 0 or args.capacity_gib < args.reserve_gib or args.reserve_gib < 4: + parser.error("invalid serial profile or explicit memory budget") + + from vllm_omni.config.stage_config import ( + DeployConfig, + StageDeployConfig, + merge_pipeline_deploy, + ) + from vllm_omni.engine.stage_runtime import StageRuntime + from vllm_omni.model_executor.models.spark2_5.pipeline import SPARK2_5_GGUF_TEXT_PIPELINE + + inventory = "Here is an inventory listing. " + " ".join( + f"Item {i}: the code for warehouse district {i} is {1000 + 7 * i}." + for i in range(1, 121) + ) + " What is the code for warehouse district 73? Answer with just the number." + cases = [ + ("short", "What is the capital of France? Answer in one word.", "Paris"), + ("inventory_120", inventory, "1511"), + ] + pipeline = SPARK2_5_GGUF_TEXT_PIPELINE + backend = { + "name": "external.llamacpp.text.v1", + "model_file": str(args.model_file), + "model_sha256": args.model_sha256, + "server_bin": str(args.server_bin), + "server_sha256": args.server_sha256, + "log_file": str(args.server_log), + "device": args.device, + "expected_device_name": args.expected_device_name, + "ggml_vk_visible_devices": args.ggml_vk_visible_devices, + "context_tokens": 4096, + "max_new_tokens": 96, + "max_io_bytes": 1 << 20, + "memory_overhead_bytes": 2 << 30, + "start_timeout_s": 120, + "request_timeout_s": 120, + } + deploy = DeployConfig( + async_chunk=False, + stages=[StageDeployConfig( + stage_id=0, + backend=backend, + resource_budget={ + "capacities": {"host_ram": args.capacity_gib << 30}, + "demands": {"host_ram": args.reserve_gib << 30}, + }, + )], + ) + configs = [stage.to_omegaconf() for stage in merge_pipeline_deploy(pipeline, deploy)] + runtime = StageRuntime(configs, "local-spark-gguf", "", stage_init_timeout=120, async_chunk=False) + report = { + "scope": "complete Spark GGUF text requests through Omni StageRuntime/StagePool and external llama.cpp backend", + "model_file": str(args.model_file.resolve()), + "device": args.device, + "memory_budget": deploy.stages[0].resource_budget, + "warmup_count": args.warmups, + "measured_count": args.repeats, + } + try: + started = time.perf_counter() + runtime.initialize() + report["startup_s"] = time.perf_counter() - started + pool = runtime.stage_pools[0] + report["execution_plan"] = pool.stage_client.execution_plan + state = SimpleNamespace(sampling_params_list=[None]) + + async def request_one(name: str, prompt: str, expected: str) -> dict: + request_id = f"{name}-{time.monotonic_ns()}" + started = time.perf_counter() + await pool.submit_initial(request_id, state, {"text": prompt, "max_tokens": 96}) + deadline = time.monotonic() + 120 + while True: + output = pool.poll_graph_output(0) + if output is not None: + break + if time.monotonic() >= deadline: + raise TimeoutError("Omni llama.cpp stage request timed out") + await asyncio.sleep(0.002) + wall_s = time.perf_counter() - started + try: + if output.error: + raise RuntimeError(output.error) + answer = output.outputs[0].text.strip() + row = { + "case": name, + "answer": answer, + "expected": expected, + "correct": answer == expected, + "wall_s": wall_s, + "stage_event": output.custom_output["stage_event"], + "metrics": output.metrics, + } + if not row["correct"]: + raise RuntimeError(f"Spark answer differed: {row}") + return row + finally: + output.release_stage_buffers() + await asyncio.sleep(0) + + report["checks"] = [await request_one(*case) for case in cases] + profile_case = cases[1] + report["warmups"] = [await request_one(*profile_case) for _ in range(args.warmups)] + report["measured"] = [await request_one(*profile_case) for _ in range(args.repeats)] + values = [row["wall_s"] for row in report["measured"]] + report["nearest_rank_p50_wall_s"] = _rank(values, 0.5) + report["nearest_rank_p95_wall_s"] = _rank(values, 0.95) + if args.context_check: + try: + await pool.submit_initial( + f"overflow-{time.monotonic_ns()}", state, + {"text": "inventory " * 6000, "max_tokens": 96}, + ) + except Exception as exc: + report["context_check"] = {"rejected": True, "error": f"{type(exc).__name__}: {exc}"} + else: + report["context_check"] = {"rejected": False} + raise RuntimeError("llama.cpp accepted a prompt beyond the declared context") + report["context_check"]["recovery"] = await request_one(*cases[0]) + if args.abort_check: + request_id = f"abort-{time.monotonic_ns()}" + await pool.submit_initial( + request_id, state, + {"text": "Count from one upward and continue until the token limit.", "max_tokens": 96}, + ) + await asyncio.sleep(0.01) + await pool.abort_requests([request_id]) + await asyncio.sleep(0.1) + stale = pool.poll_graph_output(0) + report["abort_check"] = { + "stale_output": stale is not None, + "worker_exited": pool.stage_client._proc.poll() is not None, + "ledger_after_abort": runtime.resource_ledger.snapshot(), + } + if stale is not None or not report["abort_check"]["worker_exited"]: + raise RuntimeError("llama.cpp cancel did not drain worker and stale output") + report["status"] = "passed" + except BaseException as exc: + report["status"] = "failed" + report["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + runtime.shutdown() + if runtime.resource_ledger is not None: + report["ledger_after_shutdown"] = runtime.resource_ledger.snapshot() + args.output_report.parent.mkdir(parents=True, exist_ok=True) + args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps({k: v for k, v in report.items() if k not in {"measured", "warmups"}}, indent=2)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md index 6894d6c61..e40d8c53f 100644 --- a/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md @@ -1,6 +1,6 @@ # 2026-09-23 WSL CPU and native Windows expansion -This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evidence/summary/README.md) with previously unrun WSL CPU and native Windows paths. The September 22 timing tables remain historical. MiniCPM-o now has separate 20-request serial text-to-speech timing runs on WSL CPU and native Windows GPU, plus one synthetic image and one synthetic audio input on each platform; each InternVLA timing remains one observation. Spark has a 20-request Omni output-head component profile on the HX370 AMD NPU and separate 20-request **complete text** profiles through standalone llama.cpp on native Windows CPU and Radeon 890M. Qwen3-TTS has separate 20-request complete standalone text-to-WAV profiles on native Windows CPU and a CPU+Radeon 890M hybrid route. None is a sustained or full-modality profile; the standalone paths still need Omni integration. The [checkpoint manifests](../e2e_recovery_20260923/README.md) pin the earlier real weights; the [Spark NPU component record](evidence/spark_amd_npu_output_head/README.md), [CPU/Radeon whole-model record](evidence/spark_radeon890m_llamacpp/README.md), [native CPU Qwen TTS record](evidence/qwen_tts_native_cpu/README.md), and [joint CPU+Radeon TTS record](evidence/qwen_tts_radeon890m_joint/README.md) pin their respective artifacts. No model, precision or context was silently substituted. +This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evidence/summary/README.md) with previously unrun WSL CPU and native Windows paths. The September 22 timing tables remain historical. MiniCPM-o now has separate 20-request serial text-to-speech timing runs on WSL CPU and native Windows GPU, plus one synthetic image and one synthetic audio input on each platform; each InternVLA timing remains one observation. Spark has a 20-request Omni output-head component profile on the HX370 AMD NPU and separate 20-request **complete text** profiles through standalone llama.cpp on native Windows CPU and Radeon 890M. The latter now also run as bounded whole-session Omni stages, with public AsyncOmni requests checked separately. Qwen3-TTS has separate 20-request complete standalone text-to-WAV profiles on native Windows CPU and a CPU+Radeon 890M hybrid route. None is a sustained or full-modality profile; the TTS standalone paths still need Omni integration. The [checkpoint manifests](../e2e_recovery_20260923/README.md) pin the earlier real weights; the [Spark NPU component record](evidence/spark_amd_npu_output_head/README.md), [standalone CPU/Radeon whole-model record](evidence/spark_radeon890m_llamacpp/README.md), [Omni CPU/Radeon whole-session record](evidence/spark_omni_llamacpp/README.md), [native CPU Qwen TTS record](evidence/qwen_tts_native_cpu/README.md), and [joint CPU+Radeon TTS record](evidence/qwen_tts_radeon890m_joint/README.md) pin their respective artifacts. No model, precision or context was silently substituted. | Device / model | Verified depth | Observed output and time | Remaining gate | |---|---|---|---| @@ -8,6 +8,8 @@ This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evid | Native Windows HX370 CPU / Qwen3-TTS 0.6B CustomVoice, revision `85e237c12c027371202489a0ec509ded67b5e4b5` | Scoped complete standalone PyTorch text-to-WAV path (P for two named sentences; outside Omni) | Nonzero 24 kHz WAVs of 4.56 s and 3.44 s. For the first prompt, 1 warmup + 20 serial measured complete generations p50/p95 13.94/14.68 s; Whisper tiny.en ASR WER 0.20 on the first and 0.00 on the second sentence. | Omni CPU stage binding and dependency isolation, playable streaming, speech quality, memory loading peak and sustained behavior; [raw evidence](evidence/qwen_tts_native_cpu/README.md). | | Native Windows HX370 CPU / Spark-X2.5-1.7B Q4_K_M GGUF, revision `1f7fa33b1245c14730da39e125714ad3a327901b` | Scoped complete standalone llama.cpp text path (P for two named inputs; outside Omni) | Correct `Paris` and 120-item inventory answer `1511`; 1 warmup + 20 uncached serial full inventory requests p50/p95 6.65/6.86 s, 20/20 correct. Stream-close recovery passed. | Omni whole-model stage binding, broader quality, explicit loading-peak admission and sustained behavior; [raw evidence](evidence/spark_radeon890m_llamacpp/README.md). | | Native Windows HX370 Radeon 890M / Spark-X2.5-1.7B same GGUF | Scoped complete standalone llama.cpp Vulkan1 text path (P for two named inputs; outside Omni) | All 29 logged layer assignments on Vulkan1, with a separate Vulkan_Host buffer; correct `Paris` and `1511`; 1 warmup + 20 uncached serial inventory requests p50/p95 2.18/2.20 s, 20/20 correct. Stream-close recovery passed. | Omni binding, state and memory contract, broad quality, GPU-memory attribution and sustained power/thermal behavior; [raw evidence](evidence/spark_radeon890m_llamacpp/README.md). | +| Native Windows HX370 CPU / Spark-X2.5-1.7B same GGUF | Scoped complete-request Omni whole-session llama.cpp stage (P for two named inputs) | Correct `Paris` and `1511`; 1 warmup + 20 serial inventory requests p50/p95 6.93/7.05 s, 20/20 correct. CPU 0/29 GPU layers, 4 GiB explicit host reservation, cancellation drain and zero remaining ledger reservation. | Incremental output, restart after active cancellation, broad quality, memory peak and sustained use; [Omni raw evidence](evidence/spark_omni_llamacpp/README.md). | +| Native Windows HX370 Radeon 890M / Spark-X2.5-1.7B same GGUF | Scoped complete-request Omni whole-session llama.cpp Vulkan1 stage (P for two named inputs; public AsyncOmni short request checked) | Correct `Paris` and `1511`; 1 warmup + 20 serial inventory requests p50/p95 2.10/2.14 s, 20/20 correct. Radeon 29/29 GPU layers, wrong-device refusal, over-context refusal and recovery, cancellation drain and zero remaining ledger reservation. | Incremental output, restart after active cancellation, broader quality, GPU/shared-RAM peak and sustained use; [Omni raw evidence](evidence/spark_omni_llamacpp/README.md). | | Native Windows HX370 AMD NPU / Spark-X2.5-1.7B output head, revision `448e61eb392c00f2c403185c5b56d5e0665bfaab` | Real-weight RMS norm + full-logit projection component through Omni `external.graph.v1` (B+C, not P) | One fused NPU partition and seven CPU nodes; 4/4 captured activations top-1 match, 55.43–56.81 dB full-logit SNR. One warmup + 20 serial repeated-input component requests: p50/p95 10.21/12.44 ms, versus 17.97/20.29 ms for the original FP32 CPU head through Omni. NPU startup 18.21 s and peak worker RSS 3.52 GiB. [Raw evidence](evidence/spark_amd_npu_output_head/README.md). | Actual vLLM handoff and full generation, transfer cost, quality across inputs/context, memory admission, cancellation and sustained power/thermal behavior. | | WSL CPU / MiniCPM-o 4.5, revision `503e754207c94da6bb26850b4469f367c9ea3582` | Scoped complete three-stage text-to-text+speech path (P for this input only) | Coherent text and a 5.2 s mono 24 kHz WAV. First request 27.09 s after 73.65 s startup; separate 1-warmup/20-measured serial run: startup 73.89 s, request wall p50 21.86 s/p95 22.97 s. | Speech intelligibility/alignment, real image/audio quality, video input, concurrency, interruption, power and thermal behavior. | | WSL CPU / MiniCPM-o 4.5, same revision | Scoped three-stage **image**-to-text+speech path (P for one synthetic image) | For the 448×448 red square, returned a correct description and nonzero 5.2 s WAV. Startup 112.44 s; one request 37.70 s. Text and token IDs differed from WSL GPU while describing the same visible content. | Real-image suite, speech meaning, repeated latency, memory admission and clean shutdown. | diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md new file mode 100644 index 000000000..8356d8721 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md @@ -0,0 +1,24 @@ +# Spark GGUF through Omni on native Windows CPU and Radeon 890M + +This is a **scoped complete-request text path** for one pinned Spark-X2.5-1.7B Q4_K_M GGUF. The new `external.llamacpp.text.v1` client runs inside Omni `StageRuntime`/`StagePool`; a separate public `AsyncOmni.generate` request also passed with a real Spark HF config directory and the registered `spark2_5_gguf_text` pipeline. llama.cpp owns weights, KV and internal sampling. Omni owns the process lifetime, one-slot admission, declared 4 GiB host-RAM reservation, request identity, terminal text event, acknowledgement and cancellation cleanup. It uses the existing complete-request graph control path, not an ONNX graph. Incremental token streaming, restart after cancelling an active request, model-wide quality and thermal qualification are open. + +The pinned GGUF is [XHToken/Spark-X2.5-1.7B-GGUF](https://huggingface.co/XHToken/Spark-X2.5-1.7B-GGUF/tree/1f7fa33b1245c14730da39e125714ad3a327901b) revision `1f7fa33b1245c14730da39e125714ad3a327901b`, file `Spark-X2.5-1.7B-Q4_K_M.gguf` (1,107,457,856 bytes; SHA-256 `902bde2522394954ac17821b3e5fd0df02defbc6944f122253f2580acf0503f4`). The executable is llama.cpp [b11124](https://github.com/ggml-org/llama.cpp/releases/tag/b11124), build commit `1a679828f`, `llama-server.exe` SHA-256 `9ffc5919acb4cb43c7be5f3053b59014cce70a87d3a801fcad49c4f459984f52`. The artifact remains in local model storage. The model and executable hashes are checked before launch. The source/checkpoint and earlier standalone tests are in the [prior record](../spark_radeon890m_llamacpp/README.md). + +Host: Windows 11 build 26200, Ryzen AI 9 HX 370, Radeon 890M driver `32.0.22018.6001`, Performance power scheme. An RTX 5090 Laptop GPU was installed but was not selected. The native Python environment was vLLM 0.29.0 with the current Omni source. The server used one slot, 4,096 context tokens, FP16 KV, temperature zero, reasoning off, `--fit off`, `--cache-ram 0`, and per-request `cache_prompt=false`. The backend tokenizes each prompt before submission and refuses a prompt plus maximum generation bound that exceeds the declared context. It refuses a partial generation `finish_reason`. The 4 GiB admission includes the 1.11 GB model plus declared 2 GiB KV/workspace/transfer/headroom; this is an explicit bound, **not** a measured loading or shared GPU-memory peak. + +| Complete 120-item inventory request | Native CPU | Radeon 890M Vulkan1 | +|---|---:|---:| +| Measured serial requests after 1 warmup | 20/20 correct `1511` | 20/20 correct `1511` | +| Nearest-rank request-wall p50 | 6.93 s | 2.10 s | +| Nearest-rank request-wall p95 | 7.05 s | 2.14 s | +| Stage startup, including hash verification and model load | 2.56 s | 3.12 s | +| Loaded-process RSS snapshot | 1,966,784,512 B | 1,640,730,624 B | +| GPU layers offloaded in server log | 0/29 | 29/29 to Vulkan1 | + +The profile input is the fixed 120-item synthetic inventory and asks for district 73's code (`1511`); the server reports 2,463 prompt and 5 completion tokens. Both backends also answered the short `Paris` question. The [CPU JSON](omni_llamacpp_cpu_tokenbound_20260923.json) and [Radeon JSON](omni_llamacpp_radeon_tokenbound_20260923.json) retain every request wall time, output, usage, stage event, placement plan and ledger snapshot. Their [CPU server log](omni_llamacpp_cpu_tokenbound_20260923.log) and [Radeon server log](omni_llamacpp_radeon_tokenbound_20260923.log) show the hardware and real load. Timings include Omni stage submission, local HTTP and response delivery but exclude startup; they do not measure TTFT because this adapter returns complete requests. These runs are distinct from the earlier standalone 6.65/6.86 s CPU and 2.18/2.20 s Radeon profiles, so differences should not be attributed solely to Omni overhead. + +The Radeon run rejected an over-context prompt before generation, then answered `Paris` correctly. Both runs cancelled a live request, observed no stale output, terminated the owned server and released all ledger reservations without quarantine. A [final one-request Radeon smoke](omni_llamacpp_radeon_final_smoke_20260923.json) repeated the context and abort checks after the cleanup-path hardening. This proves drain and fencing for that path, not automatic backend restart or ability to continue serving after a cancelled in-flight request. An intentionally wrong expected device name was [refused](omni_llamacpp_wrong_device_20260923.json); its [log](omni_llamacpp_wrong_device_20260923.log) identifies the actual Radeon and its Vulkan1 model buffer. The [Radeon public entrypoint JSON](omni_llamacpp_radeon_public_realconfig_20260923.json) and [CPU public entrypoint JSON](omni_llamacpp_cpu_public_realconfig_20260923.json) each record `AsyncOmni.generate` returning `Paris` and a terminal text event using a real Spark HF config directory for pipeline resolution; the GGUF, rather than the HF BF16 weights, was the executed artifact. + +Reproduce with the [stage profile script](../../../../experiments/probe_omni_llamacpp_spark.py) or [public entrypoint script](../../../../experiments/probe_omni_llamacpp_entrypoint.py) from the native Windows environment. Set `PYTHONPATH` to this checkout, `PYTHONUTF8=1`, and `VLLM_ENABLE_V1_MULTIPROCESSING=0`. Supply the pinned `--model-file`, `--server-bin`, their SHA-256 arguments, `--device cpu` or `--device Vulkan1 --expected-device-name 'AMD Radeon(TM) 890M Graphics'`, and output paths. Use `--warmups 1 --repeats 20 --context-check --abort-check` for the Radeon stage profile; the CPU profile used the same warmup/repeats and abort check. The public probe additionally accepts `--model-config-dir` for the real Spark HF config. Raw driver logs retain the exact environment setup and failures. + +The acceptance boundary remains two named text inputs, one repeated serial profile, a single local worker and one short public API request. There is no broad text-quality suite, concurrency, load peak, device-memory attribution, restart-after-cancel, token streaming, or sustained power/thermal measurement. The next experiment should establish those before declaring this CPU or Radeon path generally qualified; the other 58 device × model cells are unchanged by this backend. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_public_realconfig_20260923.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_public_realconfig_20260923.driver.log new file mode 100644 index 000000000..2e095b459 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_public_realconfig_20260923.driver.log @@ -0,0 +1,51 @@ +INFO 09-23 20:22:54 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 20:22:54 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 20:22:58 [omni_base.py:216] [AsyncOmni] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 +INFO 09-23 20:22:58 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 +ERROR 09-23 20:22:58 [repo_utils.py:119] Error parsing config for \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16: The repository \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 contains custom code which must be executed to correctly load the model. You can inspect the repository content at \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 . + +ERROR 09-23 20:22:58 [repo_utils.py:119] You can inspect the repository content at https://hf.co/\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16. + +ERROR 09-23 20:22:58 [repo_utils.py:119] Please pass the argument `trust_remote_code=True` to allow custom code to be run., retrying 1 of 2 +ERROR 09-23 20:23:00 [repo_utils.py:117] Error parsing config for \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16: The repository \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 contains custom code which must be executed to correctly load the model. You can inspect the repository content at \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 . + +ERROR 09-23 20:23:00 [repo_utils.py:117] You can inspect the repository content at https://hf.co/\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16. + +ERROR 09-23 20:23:00 [repo_utils.py:117] Please pass the argument `trust_remote_code=True` to allow custom code to be run. +INFO 09-23 20:23:00 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages +INFO 09-23 20:23:01 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +INFO 09-23 20:23:01 [initialization.py:356] Loaded OmniTransferConfig with 0 connector configurations +INFO 09-23 20:23:03 [orchestrator.py:604] [Orchestrator] Starting event loop +INFO 09-23 20:23:03 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages +INFO 09-23 20:23:03 [omni_base.py:235] [AsyncOmni] AsyncOmniEngine initialized in 5.16 seconds +INFO 09-23 20:23:03 [omni_base.py:255] [AsyncOmni] Initialized with 1 stages for model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 +INFO 09-23 20:23:03 [omni_base.py:781] [AsyncOmni] Shutting down +INFO 09-23 20:23:03 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator +INFO 09-23 20:23:03 [orchestrator.py:705] [Orchestrator] Received shutdown signal +INFO 09-23 20:23:03 [orchestrator.py:3092] [Orchestrator] Shutting down all 1 client(s) +INFO 09-23 20:23:04 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down +{ + "device": "cpu", + "entrypoint": "AsyncOmni.generate", + "startup_s": 5.160100899985991, + "request_wall_s": 0.1595404000254348, + "outputs": [ + { + "request_id": "public-spark-1", + "text": "Paris", + "error": null, + "stage_event": { + "request_id": "public-spark-1-92c33ebd", + "stage_id": 0, + "epoch": 1, + "seq": 1, + "kind": "text", + "worker_generation": "08dc9cc2aafc40f79c16fbb4902ffac6", + "buffers": [], + "terminal": true, + "error": null + } + } + ], + "status": "passed" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_public_realconfig_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_public_realconfig_20260923.json new file mode 100644 index 000000000..80976f79b --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_public_realconfig_20260923.json @@ -0,0 +1,25 @@ +{ + "device": "cpu", + "entrypoint": "AsyncOmni.generate", + "startup_s": 5.160100899985991, + "request_wall_s": 0.1595404000254348, + "outputs": [ + { + "request_id": "public-spark-1", + "text": "Paris", + "error": null, + "stage_event": { + "request_id": "public-spark-1-92c33ebd", + "stage_id": 0, + "epoch": 1, + "seq": 1, + "kind": "text", + "worker_generation": "08dc9cc2aafc40f79c16fbb4902ffac6", + "buffers": [], + "terminal": true, + "error": null + } + } + ], + "status": "passed" +} \ No newline at end of file diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_public_realconfig_20260923.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_public_realconfig_20260923.log new file mode 100644 index 000000000..0678ae9f4 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_public_realconfig_20260923.log @@ -0,0 +1,212 @@ +0.00.000.736 I srv llama_server: initializing ... +0.00.449.869 I cmn common_param: common_params_print_info: build 11124 (1a679828f) with Clang 20.1.8 for Windows x86_64 +0.00.449.875 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg) +0.00.449.876 I cmn common_param: device_info: +0.00.453.542 I cmn common_param: - Vulkan0 : NVIDIA GeForce RTX 5090 Laptop GPU (24137 MiB, 23369 MiB free) +0.00.458.166 I cmn common_param: - Vulkan1 : AMD Radeon(TM) 890M Graphics (48505 MiB, 46080 MiB free) +0.00.458.175 I cmn common_param: - CPU : AMD Ryzen AI 9 HX 370 w/ Radeon 890M (64633 MiB, 30636 MiB free) +0.00.458.222 I cmn common_param: system_info: n_threads = 12 (n_threads_batch = 12) / 24 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.458.294 I srv init_listene: running without SSL +0.00.458.351 I srv init_listene: The UI is disabled +0.00.458.352 I srv init_listene: Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable +0.00.458.355 I srv init: using 23 threads for HTTP server +0.00.458.411 W srv llama_server: security: no API key is set and CORS allows all origins (see https://github.com/ggml-org/llama.cpp/pull/25655) +0.00.466.069 I srv load_model: loading model 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' +0.00.466.076 I srv load_model: local path 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' +0.00.488.768 I llama_model_loader: loaded meta data with 37 key-value pairs and 226 tensors from C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf (version GGUF V3 (latest)) +0.00.488.780 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +0.00.488.787 I llama_model_loader: - kv 0: general.architecture str = spark2_5 +0.00.488.787 I llama_model_loader: - kv 1: general.type str = model +0.00.488.791 I llama_model_loader: - kv 2: general.sampling.top_k i32 = 0 +0.00.488.798 I llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +0.00.488.799 I llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +0.00.488.800 I llama_model_loader: - kv 5: general.name str = Hf_Format +0.00.488.800 I llama_model_loader: - kv 6: general.size_label str = 1.7B +0.00.488.803 I llama_model_loader: - kv 7: spark2_5.block_count u32 = 28 +0.00.488.804 I llama_model_loader: - kv 8: spark2_5.context_length u32 = 1048576 +0.00.488.804 I llama_model_loader: - kv 9: spark2_5.embedding_length u32 = 2048 +0.00.488.804 I llama_model_loader: - kv 10: spark2_5.feed_forward_length u32 = 6656 +0.00.488.805 I llama_model_loader: - kv 11: spark2_5.attention.head_count u32 = 8 +0.00.488.805 I llama_model_loader: - kv 12: spark2_5.attention.head_count_kv u32 = 2 +0.00.488.806 I llama_model_loader: - kv 13: spark2_5.rope.freq_base f32 = 5000000.000000 +0.00.488.807 I llama_model_loader: - kv 14: spark2_5.rope.freq_base_swa f32 = 10000.000000 +0.00.488.809 I llama_model_loader: - kv 15: spark2_5.attention.layer_norm_rms_epsilon f32 = 0.000001 +0.00.488.809 I llama_model_loader: - kv 16: spark2_5.attention.key_length u32 = 256 +0.00.488.812 I llama_model_loader: - kv 17: spark2_5.attention.value_length u32 = 256 +0.00.488.813 I llama_model_loader: - kv 18: spark2_5.vocab_size u32 = 131072 +0.00.488.813 I llama_model_loader: - kv 19: spark2_5.attention.sliding_window u32 = 512 +0.00.488.833 I llama_model_loader: - kv 20: spark2_5.attention.sliding_window_pattern arr[bool,28] = [true, true, true, false, true, true,... +0.00.488.834 I llama_model_loader: - kv 21: spark2_5.rope.dimension_count u32 = 64 +0.00.488.834 I llama_model_loader: - kv 22: spark2_5.rope.dimension_count_swa u32 = 256 +0.00.488.834 I llama_model_loader: - kv 23: tokenizer.ggml.model str = gpt2 +0.00.488.835 I llama_model_loader: - kv 24: tokenizer.ggml.pre str = spark2_5 +0.00.504.572 I llama_model_loader: - kv 25: tokenizer.ggml.tokens arr[str,131072] = ["<｜start▁of▁sentence｜>", "<... +0.00.509.564 I llama_model_loader: - kv 26: tokenizer.ggml.token_type arr[i32,131072] = [3, 3, 3, 4, 4, 3, 3, 3, 4, 4, 4, 3, ... +0.00.524.977 I llama_model_loader: - kv 27: tokenizer.ggml.merges arr[str,130716] = ["Ġ Ġ", "Ġ t", "i n", "Ġ a", "Ġ... +0.00.524.982 I llama_model_loader: - kv 28: tokenizer.ggml.bos_token_id u32 = 0 +0.00.524.983 I llama_model_loader: - kv 29: tokenizer.ggml.eos_token_id u32 = 1 +0.00.524.984 I llama_model_loader: - kv 30: tokenizer.ggml.unknown_token_id u32 = 5 +0.00.524.984 I llama_model_loader: - kv 31: tokenizer.ggml.padding_token_id u32 = 2 +0.00.524.986 I llama_model_loader: - kv 32: tokenizer.ggml.add_bos_token bool = false +0.00.524.986 I llama_model_loader: - kv 33: tokenizer.ggml.add_eos_token bool = false +0.00.524.990 I llama_model_loader: - kv 34: tokenizer.chat_template str = {%- if not messages %}\n {{- raise_... +0.00.524.990 I llama_model_loader: - kv 35: general.quantization_version u32 = 2 +0.00.524.991 I llama_model_loader: - kv 36: general.file_type u32 = 15 +0.00.524.996 I llama_model_loader: - type f32: 57 tensors +0.00.524.997 I llama_model_loader: - type q4_K: 140 tensors +0.00.524.997 I llama_model_loader: - type q6_K: 29 tensors +0.00.525.002 I print_info: file format = GGUF V3 (latest) +0.00.525.002 I print_info: file type = Q4_K - Medium +0.00.525.007 I print_info: file size = 1.03 GiB (5.16 BPW) +0.00.571.983 I load: 86 unused tokens +0.00.586.771 I load: printing all EOG tokens: +0.00.586.777 I load: - 1 ('<｜end▁of▁sentence｜>') +0.00.586.898 I load: special tokens cache size = 113 +0.00.603.376 I load: token to piece cache size = 0.7975 MB +0.00.603.401 I print_info: arch = spark2_5 +0.00.603.402 I print_info: vocab_only = 0 +0.00.603.403 I print_info: no_alloc = 0 +0.00.603.403 I print_info: n_ctx_train = 1048576 +0.00.603.404 I print_info: n_embd_inp = 2048 +0.00.603.404 I print_info: n_embd = 2048 +0.00.603.404 I print_info: n_embd_out = 2048 +0.00.603.405 I print_info: n_layer = 28 +0.00.603.405 I print_info: n_layer_all = 28 +0.00.603.422 I print_info: n_head = 8 +0.00.603.423 I print_info: n_head_kv = 2 +0.00.603.423 I print_info: n_rot = 64 +0.00.603.424 I print_info: n_swa = 512 +0.00.603.424 I print_info: is_swa_any = 1 +0.00.603.424 I print_info: non_causal_type = 0 +0.00.603.425 I print_info: n_embd_head_k = 256 +0.00.603.425 I print_info: n_embd_head_v = 256 +0.00.603.426 I print_info: n_gqa = 4 +0.00.603.428 I print_info: n_embd_k_gqa = 512 +0.00.603.429 I print_info: n_embd_v_gqa = 512 +0.00.603.431 I print_info: f_norm_eps = 0.0e+00 +0.00.603.433 I print_info: f_norm_rms_eps = 1.0e-06 +0.00.603.433 I print_info: f_clamp_kqv = 0.0e+00 +0.00.603.434 I print_info: f_max_alibi_bias = 0.0e+00 +0.00.603.434 I print_info: f_logit_scale = 0.0e+00 +0.00.603.434 I print_info: f_attn_scale = 0.0e+00 +0.00.603.435 I print_info: f_attn_value_scale = 0.0000 +0.00.603.436 I print_info: n_ff = 6656 +0.00.603.437 I print_info: n_expert = 0 +0.00.603.437 I print_info: n_expert_used = 0 +0.00.603.439 I print_info: n_expert_groups = 0 +0.00.603.439 I print_info: n_group_used = 0 +0.00.603.439 I print_info: causal attn = 1 +0.00.603.439 I print_info: pooling type = -1 +0.00.603.440 I print_info: rope type = 2 +0.00.603.440 I print_info: rope scaling = linear +0.00.603.441 I print_info: freq_base_train = 5000000.0 +0.00.603.442 I print_info: freq_scale_train = 1 +0.00.603.443 I print_info: freq_base_swa = 10000.0 +0.00.603.443 I print_info: freq_scale_swa = 1 +0.00.603.444 I print_info: n_embd_head_k_swa = 256 +0.00.603.444 I print_info: n_embd_head_v_swa = 256 +0.00.603.444 I print_info: n_rot_swa = 256 +0.00.603.445 I print_info: n_ctx_orig_yarn = 1048576 +0.00.603.445 I print_info: rope_yarn_log_mul = 0.0000 +0.00.603.446 I print_info: rope_finetuned = unknown +0.00.603.446 I print_info: model type = 1.7B +0.00.603.447 I print_info: model params = 1.71 B +0.00.603.447 I print_info: general.name = Hf_Format +0.00.603.448 I print_info: vocab type = BPE +0.00.603.449 I print_info: n_vocab = 131072 +0.00.603.449 I print_info: n_merges = 130716 +0.00.603.450 I print_info: BOS token = 0 '<｜start▁of▁sentence｜>' +0.00.603.450 I print_info: EOS token = 1 '<｜end▁of▁sentence｜>' +0.00.603.451 I print_info: EOT token = 1 '<｜end▁of▁sentence｜>' +0.00.603.451 I print_info: UNK token = 5 '' +0.00.603.452 I print_info: PAD token = 2 '<｜▁pad▁｜>' +0.00.603.452 I print_info: LF token = 198 'Ċ' +0.00.603.453 I print_info: FIM PRE token = 11 '<｜fim▁begin｜>' +0.00.603.453 I print_info: FIM SUF token = 12 '<｜fim▁hole｜>' +0.00.603.454 I print_info: FIM MID token = 13 '<｜fim▁end｜>' +0.00.603.454 I print_info: EOG token = 1 '<｜end▁of▁sentence｜>' +0.00.603.454 I print_info: max token length = 210 +0.00.603.456 I load_tensors: loading model tensors, this can take a while... (load_mode = mmap) +0.00.661.065 I load_tensors: offloading 0 repeating layers to GPU +0.00.661.070 I load_tensors: offloaded 0/29 layers to GPU +0.00.661.074 I load_tensors: CPU_Mapped model buffer size = 1043.71 MiB +0.00.661.075 I load_tensors: CPU_REPACK model buffer size = 622.37 MiB +0.01.023.838 I cmn common_init_: added <｜end▁of▁sentence｜> logit bias = -inf +0.01.024.236 I llama_context: constructing llama_context +0.01.024.243 I llama_context: n_seq_max = 1 +0.01.024.243 I llama_context: n_ctx = 4096 +0.01.024.244 I llama_context: n_ctx_seq = 4096 +0.01.024.244 I llama_context: n_batch = 2048 +0.01.024.244 I llama_context: n_ubatch = 512 +0.01.024.245 I llama_context: causal_attn = 1 +0.01.024.245 I llama_context: flash_attn = auto +0.01.024.246 I llama_context: kv_unified = false +0.01.024.249 I llama_context: freq_base = 5000000.0 +0.01.024.251 I llama_context: freq_scale = 1 +0.01.024.251 I llama_context: n_rs_seq = 0 +0.01.024.251 I llama_context: n_outputs_max = 1 +0.01.024.252 I llama_context: n_outputs_max_per_seq = 1 +0.01.024.252 I llama_context: n_ctx_seq (4096) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized +0.01.024.324 I llama_context: CPU output buffer size = 0.50 MiB +0.01.024.328 I llama_kv_cache_iswa: creating non-SWA KV cache, size = 4096 cells +0.01.024.426 I llama_kv_cache: CPU KV buffer size = 56.00 MiB +0.01.027.932 I llama_kv_cache: size = 56.00 MiB ( 4096 cells, 7 layers, 1/1 seqs), K (f16): 28.00 MiB, V (f16): 28.00 MiB +0.01.027.936 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.01.027.937 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.01.027.941 I llama_kv_cache_iswa: creating SWA KV cache, size = 1024 cells +0.01.027.990 I llama_kv_cache: CPU KV buffer size = 42.00 MiB +0.01.030.588 I llama_kv_cache: size = 42.00 MiB ( 1024 cells, 21 layers, 1/1 seqs), K (f16): 21.00 MiB, V (f16): 21.00 MiB +0.01.030.592 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.01.030.593 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.01.030.596 I sched_reserve: reserving ... +0.01.031.347 I resolve_fused_ops: Flash Attention enabled +0.01.031.350 I resolve_fused_ops: resolving fused DeepSeek V4 HC support: +0.01.031.631 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled +0.01.031.893 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled +0.01.032.150 I resolve_fused_ops: fused DeepSeek V4 HC post enabled +0.01.033.194 I sched_reserve: CPU compute buffer size = 60.02 MiB +0.01.033.197 I sched_reserve: graph: nodes = 986, splits = 1, input objects = 4, input tensors = 9 +0.01.033.198 I sched_reserve: reserve took 2.60 ms, sched copies = 1 +0.01.033.277 I cmn init: llama threadpool init, n_threads = 12 +0.01.033.287 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.01.165.306 I srv load_model: initializing, n_slots = 1, n_ctx_slot = 4096, kv_unified = 'false' +0.01.165.351 I spec common_specu: no implementations specified for speculative decoding +0.01.165.353 I slot load_model: id 0 | task -1 | new slot, n_ctx = 4096 +0.01.165.494 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it +0.01.165.497 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.01.165.498 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192 +0.01.165.529 W srv init: --cache-idle-slots requires --cache-ram, disabling +0.01.181.974 I srv init: init: chat template, example_format: '<｜start▁of▁sentence｜><|System|> +you are a helpful assistant. + +You are a helpful assistant<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>Hello<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>Hi there<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>How are you?<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>' +0.01.195.270 I srv init: init: chat template, thinking = 0 +0.01.195.289 I srv init: preserve_reasoning kwarg: true +0.01.195.291 W srv init: chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve) +0.01.195.333 I srv llama_server: model loaded +0.01.195.336 I srv llama_server: listening on http://127.0.0.1:65044 +0.01.195.449 I srv update_slots: all slots are idle +0.01.406.445 I srv server_strea: conv_id= (empty=1) +0.01.406.672 I srv operator(): chat format: peg-native +0.01.406.780 I slot get_availabl: id 0 | task -1 | - skipping, slot is empty +0.01.406.785 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.01.406.827 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.01.406.842 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.01.406.845 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.01.406.855 I slot operator(): id 0 | task 0 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 28 +0.01.406.860 I slot operator(): id 0 | task 0 | cached n_tokens = 0, memory_seq_rm [0, end) +0.01.443.457 I slot operator(): id 0 | task 0 | cached n_tokens = 10, memory_seq_rm [10, end) +0.01.443.627 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.01.489.458 I slot operator(): id 0 | task 0 | cached n_tokens = 24, memory_seq_rm [24, end) +0.01.489.517 I slot init_sampler: id 0 | task 0 | init sampler, took 0.01 ms, tokens: text = 28, total = 28 +0.01.489.748 I slot create_check: id 0 | task 0 | created context checkpoint 2 of 32 (pos_min = 0, pos_max = 23, n_tokens = 24, size = 0.985 MiB) +0.01.536.076 I slot print_timing: id 0 | task 0 | prompt eval time = 107.56 ms / 28 tokens ( 3.84 ms per token, 260.32 tokens per second) +0.01.536.090 I slot print_timing: id 0 | task 0 | eval time = 21.63 ms / 2 tokens ( 21.63 ms per token, 46.24 tokens per second) +0.01.536.092 I slot print_timing: id 0 | task 0 | total time = 129.19 ms / 30 tokens +0.01.536.094 I slot print_timing: id 0 | task 0 | graphs reused = 1 +0.01.536.131 I slot release: id 0 | task 0 | stop processing: n_tokens = 29, truncated = 0 +0.01.536.146 I srv update_slots: all slots are idle diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_tokenbound_20260923.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_tokenbound_20260923.driver.log new file mode 100644 index 000000000..a144f1510 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_tokenbound_20260923.driver.log @@ -0,0 +1,136 @@ +INFO 09-23 20:16:50 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 20:16:50 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 20:16:53 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +WARNING 09-23 20:16:53 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 13] Permission denied: 'C:\\Users\\zhout\\w2' +{ + "scope": "complete Spark GGUF text requests through Omni StageRuntime/StagePool and external llama.cpp backend", + "model_file": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "device": "cpu", + "memory_budget": { + "capacities": { + "host_ram": 12884901888 + }, + "demands": { + "host_ram": 4294967296 + } + }, + "warmup_count": 1, + "measured_count": 20, + "startup_s": 2.5649227999383584, + "execution_plan": { + "backend": "external.llamacpp.text.v1", + "stage_id": 0, + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "model_sha256": "902bde2522394954ac17821b3e5fd0df02defbc6944f122253f2580acf0503f4", + "server_sha256": "9ffc5919acb4cb43c7be5f3053b59014cce70a87d3a801fcad49c4f459984f52", + "model_alias": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "requested_device": "cpu", + "expected_device_name": null, + "layer_assignments": 0, + "offloaded_layers": [ + "0", + "29" + ], + "model_buffer_devices": [], + "worker_pid": 42956, + "loaded_rss_bytes": 1966784512, + "reserved_bytes": { + "host_ram": 4294967296 + }, + "memory_pool": "host_ram", + "memory_overhead_bytes": 2147483648, + "context_tokens": 4096, + "max_new_tokens": 96, + "max_io_bytes": 1048576, + "request_capacity": 1, + "stateful_session": "llama.cpp-owned; reset for each complete request", + "evidence": "B" + }, + "checks": [ + { + "case": "short", + "answer": "Paris", + "expected": "Paris", + "correct": true, + "wall_s": 0.1545967001002282, + "stage_event": { + "request_id": "short-620550421000000", + "stage_id": 0, + "epoch": 1, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 0.1347851000027731, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 28, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.69737090007402, + "stage_event": { + "request_id": "inventory_120-620550578000000", + "stage_id": 0, + "epoch": 2, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.692776900017634, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "nearest_rank_p50_wall_s": 6.929418300045654, + "nearest_rank_p95_wall_s": 7.048734300071374, + "abort_check": { + "stale_output": false, + "worker_exited": true, + "ledger_after_abort": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } + }, + "status": "passed", + "ledger_after_shutdown": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_tokenbound_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_tokenbound_20260923.json new file mode 100644 index 000000000..397100fe5 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_tokenbound_20260923.json @@ -0,0 +1,745 @@ +{ + "scope": "complete Spark GGUF text requests through Omni StageRuntime/StagePool and external llama.cpp backend", + "model_file": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "device": "cpu", + "memory_budget": { + "capacities": { + "host_ram": 12884901888 + }, + "demands": { + "host_ram": 4294967296 + } + }, + "warmup_count": 1, + "measured_count": 20, + "startup_s": 2.5649227999383584, + "execution_plan": { + "backend": "external.llamacpp.text.v1", + "stage_id": 0, + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "model_sha256": "902bde2522394954ac17821b3e5fd0df02defbc6944f122253f2580acf0503f4", + "server_sha256": "9ffc5919acb4cb43c7be5f3053b59014cce70a87d3a801fcad49c4f459984f52", + "model_alias": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "requested_device": "cpu", + "expected_device_name": null, + "layer_assignments": 0, + "offloaded_layers": [ + "0", + "29" + ], + "model_buffer_devices": [], + "worker_pid": 42956, + "loaded_rss_bytes": 1966784512, + "reserved_bytes": { + "host_ram": 4294967296 + }, + "memory_pool": "host_ram", + "memory_overhead_bytes": 2147483648, + "context_tokens": 4096, + "max_new_tokens": 96, + "max_io_bytes": 1048576, + "request_capacity": 1, + "stateful_session": "llama.cpp-owned; reset for each complete request", + "evidence": "B" + }, + "checks": [ + { + "case": "short", + "answer": "Paris", + "expected": "Paris", + "correct": true, + "wall_s": 0.1545967001002282, + "stage_event": { + "request_id": "short-620550421000000", + "stage_id": 0, + "epoch": 1, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 0.1347851000027731, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 28, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.69737090007402, + "stage_event": { + "request_id": "inventory_120-620550578000000", + "stage_id": 0, + "epoch": 2, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.692776900017634, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "warmups": [ + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.746441899915226, + "stage_event": { + "request_id": "inventory_120-620557265000000", + "stage_id": 0, + "epoch": 3, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.722935499972664, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "measured": [ + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.839563499903306, + "stage_event": { + "request_id": "inventory_120-620564015000000", + "stage_id": 0, + "epoch": 4, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.81114050000906, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.762330499943346, + "stage_event": { + "request_id": "inventory_120-620570859000000", + "stage_id": 0, + "epoch": 5, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.736715800012462, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.8983566999668255, + "stage_event": { + "request_id": "inventory_120-620577625000000", + "stage_id": 0, + "epoch": 6, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.878025500103831, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.979827199946158, + "stage_event": { + "request_id": "inventory_120-620584515000000", + "stage_id": 0, + "epoch": 7, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.951156700029969, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.869989700033329, + "stage_event": { + "request_id": "inventory_120-620591500000000", + "stage_id": 0, + "epoch": 8, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.841536100022495, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 7.048734300071374, + "stage_event": { + "request_id": "inventory_120-620598359000000", + "stage_id": 0, + "epoch": 9, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 7.021418699994683, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 7.0649565000785515, + "stage_event": { + "request_id": "inventory_120-620605421000000", + "stage_id": 0, + "epoch": 10, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 7.037209199974313, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.928715100046247, + "stage_event": { + "request_id": "inventory_120-620612484000000", + "stage_id": 0, + "epoch": 11, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.9239478999515995, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.977977100061253, + "stage_event": { + "request_id": "inventory_120-620619406000000", + "stage_id": 0, + "epoch": 12, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.973210399970412, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.929418300045654, + "stage_event": { + "request_id": "inventory_120-620626390000000", + "stage_id": 0, + "epoch": 13, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.908103400026448, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.931300800060853, + "stage_event": { + "request_id": "inventory_120-620633312000000", + "stage_id": 0, + "epoch": 14, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.925414299941622, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.930485799908638, + "stage_event": { + "request_id": "inventory_120-620640250000000", + "stage_id": 0, + "epoch": 15, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.925165800028481, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.982897099922411, + "stage_event": { + "request_id": "inventory_120-620647171000000", + "stage_id": 0, + "epoch": 16, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.963909099926241, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.979878299986012, + "stage_event": { + "request_id": "inventory_120-620654156000000", + "stage_id": 0, + "epoch": 17, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.974149900022894, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.980959000065923, + "stage_event": { + "request_id": "inventory_120-620661140000000", + "stage_id": 0, + "epoch": 18, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.960319699952379, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.8712320000631735, + "stage_event": { + "request_id": "inventory_120-620668125000000", + "stage_id": 0, + "epoch": 19, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.864738499978557, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.847545700031333, + "stage_event": { + "request_id": "inventory_120-620675000000000", + "stage_id": 0, + "epoch": 20, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.828035900020041, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.814243200002238, + "stage_event": { + "request_id": "inventory_120-620681843000000", + "stage_id": 0, + "epoch": 21, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.7917949999682605, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.86982150003314, + "stage_event": { + "request_id": "inventory_120-620688656000000", + "stage_id": 0, + "epoch": 22, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.863579299999401, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 6.974790100008249, + "stage_event": { + "request_id": "inventory_120-620695531000000", + "stage_id": 0, + "epoch": 23, + "seq": 1, + "kind": "text", + "worker_generation": "c55d45113e8540e6956f67f8a7ab9e6b", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 6.951468299957924, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "nearest_rank_p50_wall_s": 6.929418300045654, + "nearest_rank_p95_wall_s": 7.048734300071374, + "abort_check": { + "stale_output": false, + "worker_exited": true, + "ledger_after_abort": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } + }, + "status": "passed", + "ledger_after_shutdown": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_tokenbound_20260923.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_tokenbound_20260923.log new file mode 100644 index 000000000..8aaf831cd --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_cpu_tokenbound_20260923.log @@ -0,0 +1,893 @@ +0.00.000.680 I srv llama_server: initializing ... +0.00.158.798 I cmn common_param: common_params_print_info: build 11124 (1a679828f) with Clang 20.1.8 for Windows x86_64 +0.00.158.805 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg) +0.00.158.806 I cmn common_param: device_info: +0.00.162.001 I cmn common_param: - Vulkan0 : NVIDIA GeForce RTX 5090 Laptop GPU (24137 MiB, 23369 MiB free) +0.00.165.241 I cmn common_param: - Vulkan1 : AMD Radeon(TM) 890M Graphics (48505 MiB, 46080 MiB free) +0.00.165.248 I cmn common_param: - CPU : AMD Ryzen AI 9 HX 370 w/ Radeon 890M (64633 MiB, 30646 MiB free) +0.00.165.293 I cmn common_param: system_info: n_threads = 12 (n_threads_batch = 12) / 24 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.165.363 I srv init_listene: running without SSL +0.00.165.414 I srv init_listene: The UI is disabled +0.00.165.414 I srv init_listene: Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable +0.00.165.418 I srv init: using 23 threads for HTTP server +0.00.165.476 W srv llama_server: security: no API key is set and CORS allows all origins (see https://github.com/ggml-org/llama.cpp/pull/25655) +0.00.170.773 I srv load_model: loading model 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' +0.00.170.778 I srv load_model: local path 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' +0.00.194.768 I llama_model_loader: loaded meta data with 37 key-value pairs and 226 tensors from C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf (version GGUF V3 (latest)) +0.00.194.781 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +0.00.194.787 I llama_model_loader: - kv 0: general.architecture str = spark2_5 +0.00.194.788 I llama_model_loader: - kv 1: general.type str = model +0.00.194.792 I llama_model_loader: - kv 2: general.sampling.top_k i32 = 0 +0.00.194.799 I llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +0.00.194.800 I llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +0.00.194.801 I llama_model_loader: - kv 5: general.name str = Hf_Format +0.00.194.801 I llama_model_loader: - kv 6: general.size_label str = 1.7B +0.00.194.805 I llama_model_loader: - kv 7: spark2_5.block_count u32 = 28 +0.00.194.805 I llama_model_loader: - kv 8: spark2_5.context_length u32 = 1048576 +0.00.194.806 I llama_model_loader: - kv 9: spark2_5.embedding_length u32 = 2048 +0.00.194.806 I llama_model_loader: - kv 10: spark2_5.feed_forward_length u32 = 6656 +0.00.194.806 I llama_model_loader: - kv 11: spark2_5.attention.head_count u32 = 8 +0.00.194.807 I llama_model_loader: - kv 12: spark2_5.attention.head_count_kv u32 = 2 +0.00.194.808 I llama_model_loader: - kv 13: spark2_5.rope.freq_base f32 = 5000000.000000 +0.00.194.811 I llama_model_loader: - kv 14: spark2_5.rope.freq_base_swa f32 = 10000.000000 +0.00.194.812 I llama_model_loader: - kv 15: spark2_5.attention.layer_norm_rms_epsilon f32 = 0.000001 +0.00.194.813 I llama_model_loader: - kv 16: spark2_5.attention.key_length u32 = 256 +0.00.194.813 I llama_model_loader: - kv 17: spark2_5.attention.value_length u32 = 256 +0.00.194.814 I llama_model_loader: - kv 18: spark2_5.vocab_size u32 = 131072 +0.00.194.814 I llama_model_loader: - kv 19: spark2_5.attention.sliding_window u32 = 512 +0.00.194.835 I llama_model_loader: - kv 20: spark2_5.attention.sliding_window_pattern arr[bool,28] = [true, true, true, false, true, true,... +0.00.194.836 I llama_model_loader: - kv 21: spark2_5.rope.dimension_count u32 = 64 +0.00.194.836 I llama_model_loader: - kv 22: spark2_5.rope.dimension_count_swa u32 = 256 +0.00.194.836 I llama_model_loader: - kv 23: tokenizer.ggml.model str = gpt2 +0.00.194.837 I llama_model_loader: - kv 24: tokenizer.ggml.pre str = spark2_5 +0.00.210.305 I llama_model_loader: - kv 25: tokenizer.ggml.tokens arr[str,131072] = ["<｜start▁of▁sentence｜>", "<... +0.00.215.292 I llama_model_loader: - kv 26: tokenizer.ggml.token_type arr[i32,131072] = [3, 3, 3, 4, 4, 3, 3, 3, 4, 4, 4, 3, ... +0.00.230.721 I llama_model_loader: - kv 27: tokenizer.ggml.merges arr[str,130716] = ["Ġ Ġ", "Ġ t", "i n", "Ġ a", "Ġ... +0.00.230.726 I llama_model_loader: - kv 28: tokenizer.ggml.bos_token_id u32 = 0 +0.00.230.726 I llama_model_loader: - kv 29: tokenizer.ggml.eos_token_id u32 = 1 +0.00.230.727 I llama_model_loader: - kv 30: tokenizer.ggml.unknown_token_id u32 = 5 +0.00.230.727 I llama_model_loader: - kv 31: tokenizer.ggml.padding_token_id u32 = 2 +0.00.230.728 I llama_model_loader: - kv 32: tokenizer.ggml.add_bos_token bool = false +0.00.230.728 I llama_model_loader: - kv 33: tokenizer.ggml.add_eos_token bool = false +0.00.230.733 I llama_model_loader: - kv 34: tokenizer.chat_template str = {%- if not messages %}\n {{- raise_... +0.00.230.733 I llama_model_loader: - kv 35: general.quantization_version u32 = 2 +0.00.230.734 I llama_model_loader: - kv 36: general.file_type u32 = 15 +0.00.230.740 I llama_model_loader: - type f32: 57 tensors +0.00.230.741 I llama_model_loader: - type q4_K: 140 tensors +0.00.230.741 I llama_model_loader: - type q6_K: 29 tensors +0.00.230.745 I print_info: file format = GGUF V3 (latest) +0.00.230.746 I print_info: file type = Q4_K - Medium +0.00.230.750 I print_info: file size = 1.03 GiB (5.16 BPW) +0.00.278.556 I load: 86 unused tokens +0.00.292.058 I load: printing all EOG tokens: +0.00.292.069 I load: - 1 ('<｜end▁of▁sentence｜>') +0.00.292.188 I load: special tokens cache size = 113 +0.00.307.159 I load: token to piece cache size = 0.7975 MB +0.00.307.184 I print_info: arch = spark2_5 +0.00.307.185 I print_info: vocab_only = 0 +0.00.307.185 I print_info: no_alloc = 0 +0.00.307.186 I print_info: n_ctx_train = 1048576 +0.00.307.187 I print_info: n_embd_inp = 2048 +0.00.307.187 I print_info: n_embd = 2048 +0.00.307.187 I print_info: n_embd_out = 2048 +0.00.307.188 I print_info: n_layer = 28 +0.00.307.188 I print_info: n_layer_all = 28 +0.00.307.205 I print_info: n_head = 8 +0.00.307.206 I print_info: n_head_kv = 2 +0.00.307.207 I print_info: n_rot = 64 +0.00.307.207 I print_info: n_swa = 512 +0.00.307.207 I print_info: is_swa_any = 1 +0.00.307.208 I print_info: non_causal_type = 0 +0.00.307.208 I print_info: n_embd_head_k = 256 +0.00.307.208 I print_info: n_embd_head_v = 256 +0.00.307.210 I print_info: n_gqa = 4 +0.00.307.211 I print_info: n_embd_k_gqa = 512 +0.00.307.212 I print_info: n_embd_v_gqa = 512 +0.00.307.215 I print_info: f_norm_eps = 0.0e+00 +0.00.307.216 I print_info: f_norm_rms_eps = 1.0e-06 +0.00.307.217 I print_info: f_clamp_kqv = 0.0e+00 +0.00.307.217 I print_info: f_max_alibi_bias = 0.0e+00 +0.00.307.217 I print_info: f_logit_scale = 0.0e+00 +0.00.307.218 I print_info: f_attn_scale = 0.0e+00 +0.00.307.218 I print_info: f_attn_value_scale = 0.0000 +0.00.307.220 I print_info: n_ff = 6656 +0.00.307.220 I print_info: n_expert = 0 +0.00.307.220 I print_info: n_expert_used = 0 +0.00.307.222 I print_info: n_expert_groups = 0 +0.00.307.222 I print_info: n_group_used = 0 +0.00.307.223 I print_info: causal attn = 1 +0.00.307.223 I print_info: pooling type = -1 +0.00.307.223 I print_info: rope type = 2 +0.00.307.224 I print_info: rope scaling = linear +0.00.307.225 I print_info: freq_base_train = 5000000.0 +0.00.307.226 I print_info: freq_scale_train = 1 +0.00.307.227 I print_info: freq_base_swa = 10000.0 +0.00.307.228 I print_info: freq_scale_swa = 1 +0.00.307.228 I print_info: n_embd_head_k_swa = 256 +0.00.307.228 I print_info: n_embd_head_v_swa = 256 +0.00.307.228 I print_info: n_rot_swa = 256 +0.00.307.229 I print_info: n_ctx_orig_yarn = 1048576 +0.00.307.229 I print_info: rope_yarn_log_mul = 0.0000 +0.00.307.230 I print_info: rope_finetuned = unknown +0.00.307.231 I print_info: model type = 1.7B +0.00.307.231 I print_info: model params = 1.71 B +0.00.307.232 I print_info: general.name = Hf_Format +0.00.307.233 I print_info: vocab type = BPE +0.00.307.233 I print_info: n_vocab = 131072 +0.00.307.233 I print_info: n_merges = 130716 +0.00.307.234 I print_info: BOS token = 0 '<｜start▁of▁sentence｜>' +0.00.307.234 I print_info: EOS token = 1 '<｜end▁of▁sentence｜>' +0.00.307.235 I print_info: EOT token = 1 '<｜end▁of▁sentence｜>' +0.00.307.235 I print_info: UNK token = 5 '' +0.00.307.236 I print_info: PAD token = 2 '<｜▁pad▁｜>' +0.00.307.236 I print_info: LF token = 198 'Ċ' +0.00.307.237 I print_info: FIM PRE token = 11 '<｜fim▁begin｜>' +0.00.307.237 I print_info: FIM SUF token = 12 '<｜fim▁hole｜>' +0.00.307.238 I print_info: FIM MID token = 13 '<｜fim▁end｜>' +0.00.307.238 I print_info: EOG token = 1 '<｜end▁of▁sentence｜>' +0.00.307.239 I print_info: max token length = 210 +0.00.307.241 I load_tensors: loading model tensors, this can take a while... (load_mode = mmap) +0.00.366.452 I load_tensors: offloading 0 repeating layers to GPU +0.00.366.459 I load_tensors: offloaded 0/29 layers to GPU +0.00.366.464 I load_tensors: CPU_Mapped model buffer size = 1043.71 MiB +0.00.366.465 I load_tensors: CPU_REPACK model buffer size = 622.37 MiB +0.00.720.667 I cmn common_init_: added <｜end▁of▁sentence｜> logit bias = -inf +0.00.721.039 I llama_context: constructing llama_context +0.00.721.046 I llama_context: n_seq_max = 1 +0.00.721.046 I llama_context: n_ctx = 4096 +0.00.721.047 I llama_context: n_ctx_seq = 4096 +0.00.721.047 I llama_context: n_batch = 2048 +0.00.721.047 I llama_context: n_ubatch = 512 +0.00.721.048 I llama_context: causal_attn = 1 +0.00.721.048 I llama_context: flash_attn = auto +0.00.721.049 I llama_context: kv_unified = false +0.00.721.052 I llama_context: freq_base = 5000000.0 +0.00.721.054 I llama_context: freq_scale = 1 +0.00.721.055 I llama_context: n_rs_seq = 0 +0.00.721.055 I llama_context: n_outputs_max = 1 +0.00.721.055 I llama_context: n_outputs_max_per_seq = 1 +0.00.721.056 I llama_context: n_ctx_seq (4096) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized +0.00.721.141 I llama_context: CPU output buffer size = 0.50 MiB +0.00.721.147 I llama_kv_cache_iswa: creating non-SWA KV cache, size = 4096 cells +0.00.721.250 I llama_kv_cache: CPU KV buffer size = 56.00 MiB +0.00.725.017 I llama_kv_cache: size = 56.00 MiB ( 4096 cells, 7 layers, 1/1 seqs), K (f16): 28.00 MiB, V (f16): 28.00 MiB +0.00.725.025 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.00.725.026 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.00.725.030 I llama_kv_cache_iswa: creating SWA KV cache, size = 1024 cells +0.00.725.118 I llama_kv_cache: CPU KV buffer size = 42.00 MiB +0.00.727.832 I llama_kv_cache: size = 42.00 MiB ( 1024 cells, 21 layers, 1/1 seqs), K (f16): 21.00 MiB, V (f16): 21.00 MiB +0.00.727.838 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.00.727.839 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.00.727.843 I sched_reserve: reserving ... +0.00.728.735 I resolve_fused_ops: Flash Attention enabled +0.00.728.738 I resolve_fused_ops: resolving fused DeepSeek V4 HC support: +0.00.729.063 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled +0.00.729.347 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled +0.00.729.618 I resolve_fused_ops: fused DeepSeek V4 HC post enabled +0.00.730.741 I sched_reserve: CPU compute buffer size = 60.02 MiB +0.00.730.745 I sched_reserve: graph: nodes = 986, splits = 1, input objects = 4, input tensors = 9 +0.00.730.746 I sched_reserve: reserve took 2.90 ms, sched copies = 1 +0.00.730.831 I cmn init: llama threadpool init, n_threads = 12 +0.00.730.844 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.00.857.157 I srv load_model: initializing, n_slots = 1, n_ctx_slot = 4096, kv_unified = 'false' +0.00.857.198 I spec common_specu: no implementations specified for speculative decoding +0.00.857.200 I slot load_model: id 0 | task -1 | new slot, n_ctx = 4096 +0.00.857.341 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it +0.00.857.344 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.00.857.344 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192 +0.00.857.372 W srv init: --cache-idle-slots requires --cache-ram, disabling +0.00.874.690 I srv init: init: chat template, example_format: '<｜start▁of▁sentence｜><|System|> +you are a helpful assistant. + +You are a helpful assistant<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>Hello<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>Hi there<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>How are you?<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>' +0.00.885.641 I srv init: init: chat template, thinking = 0 +0.00.885.653 I srv init: preserve_reasoning kwarg: true +0.00.885.654 W srv init: chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve) +0.00.885.683 I srv llama_server: model loaded +0.00.885.685 I srv llama_server: listening on http://127.0.0.1:61319 +0.00.885.790 I srv update_slots: all slots are idle +0.01.016.594 I srv server_strea: conv_id= (empty=1) +0.01.016.778 I srv operator(): chat format: peg-native +0.01.016.920 I slot get_availabl: id 0 | task -1 | - skipping, slot is empty +0.01.016.930 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.01.016.996 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.01.017.018 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.01.017.023 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.01.017.039 I slot operator(): id 0 | task 0 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 28 +0.01.017.046 I slot operator(): id 0 | task 0 | cached n_tokens = 0, memory_seq_rm [0, end) +0.01.050.127 I slot operator(): id 0 | task 0 | cached n_tokens = 10, memory_seq_rm [10, end) +0.01.050.324 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.01.094.703 I slot operator(): id 0 | task 0 | cached n_tokens = 24, memory_seq_rm [24, end) +0.01.094.765 I slot init_sampler: id 0 | task 0 | init sampler, took 0.01 ms, tokens: text = 28, total = 28 +0.01.095.022 I slot create_check: id 0 | task 0 | created context checkpoint 2 of 32 (pos_min = 0, pos_max = 23, n_tokens = 24, size = 0.985 MiB) +0.01.138.442 I slot print_timing: id 0 | task 0 | prompt eval time = 101.89 ms / 28 tokens ( 3.64 ms per token, 274.79 tokens per second) +0.01.138.451 I slot print_timing: id 0 | task 0 | eval time = 19.48 ms / 2 tokens ( 19.48 ms per token, 51.33 tokens per second) +0.01.138.452 I slot print_timing: id 0 | task 0 | total time = 121.38 ms / 30 tokens +0.01.138.453 I slot print_timing: id 0 | task 0 | graphs reused = 1 +0.01.138.498 I slot release: id 0 | task 0 | stop processing: n_tokens = 29, truncated = 0 +0.01.138.507 I srv update_slots: all slots are idle +0.01.169.641 I srv server_strea: conv_id= (empty=1) +0.01.171.742 I srv operator(): chat format: peg-native +0.01.171.882 I slot get_availabl: id 0 | task -1 | - checking sim = 0.005 (12/2463) > 0.100 +0.01.171.889 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = 981683 +0.01.172.135 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.01.172.152 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.01.172.154 I slot launch_slot_: id 0 | task 5 | processing task, is_child = 0 +0.01.172.165 I slot operator(): id 0 | task 5 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.01.172.169 I slot operator(): id 0 | task 5 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.01.172.171 I slot operator(): id 0 | task 5 | erased invalidated context checkpoint (pos_min = 0, pos_max = 23, n_tokens = 24, n_swa = 512, pos_next = 0, size = 0.985 MiB) +0.01.172.173 I slot operator(): id 0 | task 5 | cached n_tokens = 0, memory_seq_rm [0, end) +0.01.207.089 I slot operator(): id 0 | task 5 | cached n_tokens = 10, memory_seq_rm [10, end) +0.01.207.235 I slot create_check: id 0 | task 5 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.06.284.691 I slot print_timing: id 0 | task 5 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.11 s / 380.83 tokens per second +0.06.284.698 I slot operator(): id 0 | task 5 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.06.288.812 I slot create_check: id 0 | task 5 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.07.714.677 I slot print_timing: id 0 | task 5 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.54 s / 375.85 tokens per second +0.07.714.684 I slot operator(): id 0 | task 5 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.07.715.041 I slot init_sampler: id 0 | task 5 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.07.719.205 I slot create_check: id 0 | task 5 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.07.836.021 I slot print_timing: id 0 | task 5 | prompt eval time = 6578.05 ms / 2463 tokens ( 2.67 ms per token, 374.43 tokens per second) +0.07.836.032 I slot print_timing: id 0 | task 5 | eval time = 85.79 ms / 5 tokens ( 21.45 ms per token, 46.63 tokens per second) +0.07.836.033 I slot print_timing: id 0 | task 5 | total time = 6663.84 ms / 2468 tokens +0.07.836.034 I slot print_timing: id 0 | task 5 | graphs reused = 4 +0.07.836.084 I slot release: id 0 | task 5 | stop processing: n_tokens = 2467, truncated = 0 +0.07.836.099 I srv update_slots: all slots are idle +0.07.889.254 I srv server_strea: conv_id= (empty=1) +0.07.892.406 I srv operator(): chat format: peg-native +0.07.892.610 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.07.892.621 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.07.892.850 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.07.892.866 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.07.892.868 I slot launch_slot_: id 0 | task 14 | processing task, is_child = 0 +0.07.892.877 I slot operator(): id 0 | task 14 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.07.892.880 I slot operator(): id 0 | task 14 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.07.892.883 I slot operator(): id 0 | task 14 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.07.893.743 I slot operator(): id 0 | task 14 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.07.894.538 I slot operator(): id 0 | task 14 | cached n_tokens = 0, memory_seq_rm [0, end) +0.07.930.214 I slot operator(): id 0 | task 14 | cached n_tokens = 10, memory_seq_rm [10, end) +0.07.930.363 I slot create_check: id 0 | task 14 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.13.004.735 I slot print_timing: id 0 | task 14 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.11 s / 380.88 tokens per second +0.13.004.746 I slot operator(): id 0 | task 14 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.13.009.085 I slot create_check: id 0 | task 14 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.14.453.801 I slot print_timing: id 0 | task 14 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.56 s / 374.80 tokens per second +0.14.453.812 I slot operator(): id 0 | task 14 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.14.454.171 I slot init_sampler: id 0 | task 14 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.14.458.046 I slot create_check: id 0 | task 14 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.14.582.575 I slot print_timing: id 0 | task 14 | prompt eval time = 6598.28 ms / 2463 tokens ( 2.68 ms per token, 373.28 tokens per second) +0.14.582.583 I slot print_timing: id 0 | task 14 | eval time = 91.41 ms / 5 tokens ( 22.85 ms per token, 43.76 tokens per second) +0.14.582.584 I slot print_timing: id 0 | task 14 | total time = 6689.68 ms / 2468 tokens +0.14.582.585 I slot print_timing: id 0 | task 14 | graphs reused = 7 +0.14.582.630 I slot release: id 0 | task 14 | stop processing: n_tokens = 2467, truncated = 0 +0.14.582.646 I srv update_slots: all slots are idle +0.14.649.388 I srv server_strea: conv_id= (empty=1) +0.14.651.708 I srv operator(): chat format: peg-native +0.14.651.908 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.14.651.919 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.14.652.257 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.14.652.280 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.14.652.283 I slot launch_slot_: id 0 | task 23 | processing task, is_child = 0 +0.14.652.294 I slot operator(): id 0 | task 23 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.14.652.299 I slot operator(): id 0 | task 23 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.14.652.304 I slot operator(): id 0 | task 23 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.14.653.312 I slot operator(): id 0 | task 23 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.14.653.917 I slot operator(): id 0 | task 23 | cached n_tokens = 0, memory_seq_rm [0, end) +0.14.688.828 I slot operator(): id 0 | task 23 | cached n_tokens = 10, memory_seq_rm [10, end) +0.14.688.957 I slot create_check: id 0 | task 23 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.19.850.857 I slot print_timing: id 0 | task 23 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.20 s / 374.53 tokens per second +0.19.850.868 I slot operator(): id 0 | task 23 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.19.855.085 I slot create_check: id 0 | task 23 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.21.295.750 I slot print_timing: id 0 | task 23 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.64 s / 370.14 tokens per second +0.21.295.769 I slot operator(): id 0 | task 23 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.21.296.314 I slot init_sampler: id 0 | task 23 | init sampler, took 0.52 ms, tokens: text = 2463, total = 2463 +0.21.300.227 I slot create_check: id 0 | task 23 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.21.422.107 I slot print_timing: id 0 | task 23 | prompt eval time = 6679.31 ms / 2463 tokens ( 2.71 ms per token, 368.75 tokens per second) +0.21.422.130 I slot print_timing: id 0 | task 23 | eval time = 90.48 ms / 5 tokens ( 22.62 ms per token, 44.21 tokens per second) +0.21.422.133 I slot print_timing: id 0 | task 23 | total time = 6769.79 ms / 2468 tokens +0.21.422.135 I slot print_timing: id 0 | task 23 | graphs reused = 10 +0.21.422.203 I slot release: id 0 | task 23 | stop processing: n_tokens = 2467, truncated = 0 +0.21.422.228 I srv update_slots: all slots are idle +0.21.474.086 I srv server_strea: conv_id= (empty=1) +0.21.476.429 I srv operator(): chat format: peg-native +0.21.476.604 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.21.476.614 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.21.476.870 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.21.476.885 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.21.476.888 I slot launch_slot_: id 0 | task 32 | processing task, is_child = 0 +0.21.476.896 I slot operator(): id 0 | task 32 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.21.476.900 I slot operator(): id 0 | task 32 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.21.476.903 I slot operator(): id 0 | task 32 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.21.477.784 I slot operator(): id 0 | task 32 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.21.478.605 I slot operator(): id 0 | task 32 | cached n_tokens = 0, memory_seq_rm [0, end) +0.21.515.084 I slot operator(): id 0 | task 32 | cached n_tokens = 10, memory_seq_rm [10, end) +0.21.515.257 I slot create_check: id 0 | task 32 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.26.611.690 I slot print_timing: id 0 | task 32 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.13 s / 379.18 tokens per second +0.26.611.699 I slot operator(): id 0 | task 32 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.26.615.666 I slot create_check: id 0 | task 32 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.28.053.866 I slot print_timing: id 0 | task 32 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.58 s / 373.88 tokens per second +0.28.053.873 I slot operator(): id 0 | task 32 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.28.054.237 I slot init_sampler: id 0 | task 32 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.28.058.229 I slot create_check: id 0 | task 32 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.28.184.357 I slot print_timing: id 0 | task 32 | prompt eval time = 6613.50 ms / 2463 tokens ( 2.69 ms per token, 372.42 tokens per second) +0.28.184.367 I slot print_timing: id 0 | task 32 | eval time = 93.94 ms / 5 tokens ( 23.48 ms per token, 42.58 tokens per second) +0.28.184.368 I slot print_timing: id 0 | task 32 | total time = 6707.44 ms / 2468 tokens +0.28.184.369 I slot print_timing: id 0 | task 32 | graphs reused = 13 +0.28.184.435 I slot release: id 0 | task 32 | stop processing: n_tokens = 2467, truncated = 0 +0.28.184.454 I srv update_slots: all slots are idle +0.28.220.221 I srv server_strea: conv_id= (empty=1) +0.28.222.790 I srv operator(): chat format: peg-native +0.28.223.017 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.28.223.030 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.28.223.378 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.28.223.404 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.28.223.408 I slot launch_slot_: id 0 | task 41 | processing task, is_child = 0 +0.28.223.419 I slot operator(): id 0 | task 41 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.28.223.424 I slot operator(): id 0 | task 41 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.28.223.429 I slot operator(): id 0 | task 41 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.28.224.329 I slot operator(): id 0 | task 41 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.28.225.137 I slot operator(): id 0 | task 41 | cached n_tokens = 0, memory_seq_rm [0, end) +0.28.261.178 I slot operator(): id 0 | task 41 | cached n_tokens = 10, memory_seq_rm [10, end) +0.28.261.304 I slot create_check: id 0 | task 41 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.33.480.711 I slot print_timing: id 0 | task 41 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.26 s / 370.35 tokens per second +0.33.480.722 I slot operator(): id 0 | task 41 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.33.485.384 I slot create_check: id 0 | task 41 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.34.956.437 I slot print_timing: id 0 | task 41 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.73 s / 365.22 tokens per second +0.34.956.444 I slot operator(): id 0 | task 41 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.34.956.800 I slot init_sampler: id 0 | task 41 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.34.960.827 I slot create_check: id 0 | task 41 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.35.083.121 I slot print_timing: id 0 | task 41 | prompt eval time = 6769.60 ms / 2463 tokens ( 2.75 ms per token, 363.83 tokens per second) +0.35.083.130 I slot print_timing: id 0 | task 41 | eval time = 90.09 ms / 5 tokens ( 22.52 ms per token, 44.40 tokens per second) +0.35.083.131 I slot print_timing: id 0 | task 41 | total time = 6859.69 ms / 2468 tokens +0.35.083.132 I slot print_timing: id 0 | task 41 | graphs reused = 16 +0.35.083.193 I slot release: id 0 | task 41 | stop processing: n_tokens = 2467, truncated = 0 +0.35.083.219 I srv update_slots: all slots are idle +0.35.157.245 I srv server_strea: conv_id= (empty=1) +0.35.160.837 I srv operator(): chat format: peg-native +0.35.161.061 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.35.161.075 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.35.161.360 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.35.161.376 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.35.161.378 I slot launch_slot_: id 0 | task 50 | processing task, is_child = 0 +0.35.161.386 I slot operator(): id 0 | task 50 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.35.161.390 I slot operator(): id 0 | task 50 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.35.161.393 I slot operator(): id 0 | task 50 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.35.162.218 I slot operator(): id 0 | task 50 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.35.163.101 I slot operator(): id 0 | task 50 | cached n_tokens = 0, memory_seq_rm [0, end) +0.35.199.648 I slot operator(): id 0 | task 50 | cached n_tokens = 10, memory_seq_rm [10, end) +0.35.199.774 I slot create_check: id 0 | task 50 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.40.439.935 I slot print_timing: id 0 | task 50 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.28 s / 368.85 tokens per second +0.40.439.946 I slot operator(): id 0 | task 50 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.40.443.989 I slot create_check: id 0 | task 50 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.41.933.361 I slot print_timing: id 0 | task 50 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.77 s / 363.12 tokens per second +0.41.933.370 I slot operator(): id 0 | task 50 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.41.933.672 I slot init_sampler: id 0 | task 50 | init sampler, took 0.28 ms, tokens: text = 2463, total = 2463 +0.41.937.471 I slot create_check: id 0 | task 50 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.42.062.851 I slot print_timing: id 0 | task 50 | prompt eval time = 6807.72 ms / 2463 tokens ( 2.76 ms per token, 361.80 tokens per second) +0.42.062.866 I slot print_timing: id 0 | task 50 | eval time = 93.73 ms / 5 tokens ( 23.43 ms per token, 42.68 tokens per second) +0.42.062.867 I slot print_timing: id 0 | task 50 | total time = 6901.45 ms / 2468 tokens +0.42.062.869 I slot print_timing: id 0 | task 50 | graphs reused = 19 +0.42.062.939 I slot release: id 0 | task 50 | stop processing: n_tokens = 2467, truncated = 0 +0.42.062.961 I srv update_slots: all slots are idle +0.42.125.565 I srv server_strea: conv_id= (empty=1) +0.42.129.120 I srv operator(): chat format: peg-native +0.42.129.341 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.42.129.353 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.42.129.620 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.42.129.638 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.42.129.641 I slot launch_slot_: id 0 | task 59 | processing task, is_child = 0 +0.42.129.650 I slot operator(): id 0 | task 59 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.42.129.654 I slot operator(): id 0 | task 59 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.42.129.658 I slot operator(): id 0 | task 59 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.42.130.564 I slot operator(): id 0 | task 59 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.42.131.449 I slot operator(): id 0 | task 59 | cached n_tokens = 0, memory_seq_rm [0, end) +0.42.168.134 I slot operator(): id 0 | task 59 | cached n_tokens = 10, memory_seq_rm [10, end) +0.42.168.236 I slot create_check: id 0 | task 59 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.47.360.816 I slot print_timing: id 0 | task 59 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.23 s / 372.19 tokens per second +0.47.360.822 I slot operator(): id 0 | task 59 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.47.364.564 I slot create_check: id 0 | task 59 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.48.804.091 I slot print_timing: id 0 | task 59 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.67 s / 368.42 tokens per second +0.48.804.105 I slot operator(): id 0 | task 59 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.48.804.691 I slot init_sampler: id 0 | task 59 | init sampler, took 0.56 ms, tokens: text = 2463, total = 2463 +0.48.808.784 I slot create_check: id 0 | task 59 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.48.932.964 I slot print_timing: id 0 | task 59 | prompt eval time = 6710.32 ms / 2463 tokens ( 2.72 ms per token, 367.05 tokens per second) +0.48.932.973 I slot print_timing: id 0 | task 59 | eval time = 92.98 ms / 5 tokens ( 23.24 ms per token, 43.02 tokens per second) +0.48.932.975 I slot print_timing: id 0 | task 59 | total time = 6803.30 ms / 2468 tokens +0.48.932.976 I slot print_timing: id 0 | task 59 | graphs reused = 22 +0.48.933.045 I slot release: id 0 | task 59 | stop processing: n_tokens = 2467, truncated = 0 +0.48.933.063 I srv update_slots: all slots are idle +0.48.989.602 I srv server_strea: conv_id= (empty=1) +0.48.992.605 I srv operator(): chat format: peg-native +0.48.992.783 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.48.992.790 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.48.993.038 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.48.993.053 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.48.993.055 I slot launch_slot_: id 0 | task 68 | processing task, is_child = 0 +0.48.993.064 I slot operator(): id 0 | task 68 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.48.993.067 I slot operator(): id 0 | task 68 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.48.993.070 I slot operator(): id 0 | task 68 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.48.993.884 I slot operator(): id 0 | task 68 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.48.994.702 I slot operator(): id 0 | task 68 | cached n_tokens = 0, memory_seq_rm [0, end) +0.49.031.485 I slot operator(): id 0 | task 68 | cached n_tokens = 10, memory_seq_rm [10, end) +0.49.031.602 I slot create_check: id 0 | task 68 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.54.311.994 I slot print_timing: id 0 | task 68 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.32 s / 366.05 tokens per second +0.54.312.002 I slot operator(): id 0 | task 68 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.54.316.528 I slot create_check: id 0 | task 68 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.55.850.272 I slot print_timing: id 0 | task 68 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.86 s / 358.60 tokens per second +0.55.850.282 I slot operator(): id 0 | task 68 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.55.850.647 I slot init_sampler: id 0 | task 68 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.55.854.805 I slot create_check: id 0 | task 68 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.55.981.865 I slot print_timing: id 0 | task 68 | prompt eval time = 6894.43 ms / 2463 tokens ( 2.80 ms per token, 357.25 tokens per second) +0.55.981.879 I slot print_timing: id 0 | task 68 | eval time = 94.35 ms / 5 tokens ( 23.59 ms per token, 42.39 tokens per second) +0.55.981.881 I slot print_timing: id 0 | task 68 | total time = 6988.78 ms / 2468 tokens +0.55.981.882 I slot print_timing: id 0 | task 68 | graphs reused = 25 +0.55.981.947 I slot release: id 0 | task 68 | stop processing: n_tokens = 2467, truncated = 0 +0.55.981.976 I srv update_slots: all slots are idle +0.56.026.989 I srv server_strea: conv_id= (empty=1) +0.56.029.383 I srv operator(): chat format: peg-native +0.56.029.569 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.56.029.579 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.56.029.822 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.56.029.837 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.56.029.840 I slot launch_slot_: id 0 | task 77 | processing task, is_child = 0 +0.56.029.849 I slot operator(): id 0 | task 77 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.56.029.852 I slot operator(): id 0 | task 77 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.56.029.855 I slot operator(): id 0 | task 77 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.56.030.667 I slot operator(): id 0 | task 77 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.56.031.473 I slot operator(): id 0 | task 77 | cached n_tokens = 0, memory_seq_rm [0, end) +0.56.065.674 I slot operator(): id 0 | task 77 | cached n_tokens = 10, memory_seq_rm [10, end) +0.56.065.806 I slot create_check: id 0 | task 77 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +1.01.400.671 I slot print_timing: id 0 | task 77 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.37 s / 362.52 tokens per second +1.01.400.679 I slot operator(): id 0 | task 77 | cached n_tokens = 1947, memory_seq_rm [1947, end) +1.01.404.522 I slot create_check: id 0 | task 77 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +1.02.894.608 I slot print_timing: id 0 | task 77 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.86 s / 358.21 tokens per second +1.02.894.618 I slot operator(): id 0 | task 77 | cached n_tokens = 2459, memory_seq_rm [2459, end) +1.02.894.897 I slot init_sampler: id 0 | task 77 | init sampler, took 0.26 ms, tokens: text = 2463, total = 2463 +1.02.898.615 I slot create_check: id 0 | task 77 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +1.03.046.942 I slot print_timing: id 0 | task 77 | prompt eval time = 6902.12 ms / 2463 tokens ( 2.80 ms per token, 356.85 tokens per second) +1.03.046.952 I slot print_timing: id 0 | task 77 | eval time = 114.96 ms / 5 tokens ( 28.74 ms per token, 34.80 tokens per second) +1.03.046.953 I slot print_timing: id 0 | task 77 | total time = 7017.07 ms / 2468 tokens +1.03.046.955 I slot print_timing: id 0 | task 77 | graphs reused = 28 +1.03.047.017 I slot release: id 0 | task 77 | stop processing: n_tokens = 2467, truncated = 0 +1.03.047.039 I srv update_slots: all slots are idle +1.03.067.182 I srv server_strea: conv_id= (empty=1) +1.03.069.372 I srv operator(): chat format: peg-native +1.03.069.538 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +1.03.069.547 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +1.03.069.793 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.03.069.808 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.03.069.810 I slot launch_slot_: id 0 | task 86 | processing task, is_child = 0 +1.03.069.818 I slot operator(): id 0 | task 86 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +1.03.069.821 I slot operator(): id 0 | task 86 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +1.03.069.825 I slot operator(): id 0 | task 86 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.03.070.791 I slot operator(): id 0 | task 86 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.03.071.651 I slot operator(): id 0 | task 86 | cached n_tokens = 0, memory_seq_rm [0, end) +1.03.106.284 I slot operator(): id 0 | task 86 | cached n_tokens = 10, memory_seq_rm [10, end) +1.03.106.452 I slot create_check: id 0 | task 86 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +1.08.374.268 I slot print_timing: id 0 | task 86 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.30 s / 367.05 tokens per second +1.08.374.275 I slot operator(): id 0 | task 86 | cached n_tokens = 1947, memory_seq_rm [1947, end) +1.08.378.539 I slot create_check: id 0 | task 86 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +1.09.848.013 I slot print_timing: id 0 | task 86 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.78 s / 362.78 tokens per second +1.09.848.023 I slot operator(): id 0 | task 86 | cached n_tokens = 2459, memory_seq_rm [2459, end) +1.09.848.380 I slot init_sampler: id 0 | task 86 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +1.09.852.537 I slot create_check: id 0 | task 86 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +1.09.975.911 I slot print_timing: id 0 | task 86 | prompt eval time = 6814.41 ms / 2463 tokens ( 2.77 ms per token, 361.44 tokens per second) +1.09.975.920 I slot print_timing: id 0 | task 86 | eval time = 91.66 ms / 5 tokens ( 22.92 ms per token, 43.64 tokens per second) +1.09.975.921 I slot print_timing: id 0 | task 86 | total time = 6906.07 ms / 2468 tokens +1.09.975.922 I slot print_timing: id 0 | task 86 | graphs reused = 31 +1.09.975.970 I slot release: id 0 | task 86 | stop processing: n_tokens = 2467, truncated = 0 +1.09.975.994 I srv update_slots: all slots are idle +1.09.998.963 I srv server_strea: conv_id= (empty=1) +1.10.002.074 I srv operator(): chat format: peg-native +1.10.002.322 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +1.10.002.329 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +1.10.002.565 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.10.002.579 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.10.002.582 I slot launch_slot_: id 0 | task 95 | processing task, is_child = 0 +1.10.002.588 I slot operator(): id 0 | task 95 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +1.10.002.591 I slot operator(): id 0 | task 95 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +1.10.002.594 I slot operator(): id 0 | task 95 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.10.003.327 I slot operator(): id 0 | task 95 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.10.004.029 I slot operator(): id 0 | task 95 | cached n_tokens = 0, memory_seq_rm [0, end) +1.10.040.257 I slot operator(): id 0 | task 95 | cached n_tokens = 10, memory_seq_rm [10, end) +1.10.040.417 I slot create_check: id 0 | task 95 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +1.15.325.364 I slot print_timing: id 0 | task 95 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.32 s / 365.79 tokens per second +1.15.325.372 I slot operator(): id 0 | task 95 | cached n_tokens = 1947, memory_seq_rm [1947, end) +1.15.328.825 I slot create_check: id 0 | task 95 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +1.16.827.094 I slot print_timing: id 0 | task 95 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.82 s / 360.32 tokens per second +1.16.827.111 I slot operator(): id 0 | task 95 | cached n_tokens = 2459, memory_seq_rm [2459, end) +1.16.827.699 I slot init_sampler: id 0 | task 95 | init sampler, took 0.55 ms, tokens: text = 2463, total = 2463 +1.16.831.597 I slot create_check: id 0 | task 95 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +1.16.954.077 I slot print_timing: id 0 | task 95 | prompt eval time = 6860.08 ms / 2463 tokens ( 2.79 ms per token, 359.03 tokens per second) +1.16.954.092 I slot print_timing: id 0 | task 95 | eval time = 91.39 ms / 5 tokens ( 22.85 ms per token, 43.77 tokens per second) +1.16.954.094 I slot print_timing: id 0 | task 95 | total time = 6951.47 ms / 2468 tokens +1.16.954.095 I slot print_timing: id 0 | task 95 | graphs reused = 34 +1.16.954.152 I slot release: id 0 | task 95 | stop processing: n_tokens = 2467, truncated = 0 +1.16.954.170 I srv update_slots: all slots are idle +1.17.000.923 I srv server_strea: conv_id= (empty=1) +1.17.003.678 I srv operator(): chat format: peg-native +1.17.003.901 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +1.17.003.912 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +1.17.004.178 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.17.004.194 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.17.004.197 I slot launch_slot_: id 0 | task 104 | processing task, is_child = 0 +1.17.004.205 I slot operator(): id 0 | task 104 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +1.17.004.209 I slot operator(): id 0 | task 104 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +1.17.004.212 I slot operator(): id 0 | task 104 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.17.005.143 I slot operator(): id 0 | task 104 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.17.006.039 I slot operator(): id 0 | task 104 | cached n_tokens = 0, memory_seq_rm [0, end) +1.17.041.950 I slot operator(): id 0 | task 104 | cached n_tokens = 10, memory_seq_rm [10, end) +1.17.042.118 I slot create_check: id 0 | task 104 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +1.22.279.034 I slot print_timing: id 0 | task 104 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.27 s / 369.11 tokens per second +1.22.279.042 I slot operator(): id 0 | task 104 | cached n_tokens = 1947, memory_seq_rm [1947, end) +1.22.283.018 I slot create_check: id 0 | task 104 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +1.23.751.563 I slot print_timing: id 0 | task 104 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.75 s / 364.44 tokens per second +1.23.751.572 I slot operator(): id 0 | task 104 | cached n_tokens = 2459, memory_seq_rm [2459, end) +1.23.751.876 I slot init_sampler: id 0 | task 104 | init sampler, took 0.28 ms, tokens: text = 2463, total = 2463 +1.23.755.718 I slot create_check: id 0 | task 104 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +1.23.883.256 I slot print_timing: id 0 | task 104 | prompt eval time = 6784.70 ms / 2463 tokens ( 2.75 ms per token, 363.02 tokens per second) +1.23.883.272 I slot print_timing: id 0 | task 104 | eval time = 94.33 ms / 5 tokens ( 23.58 ms per token, 42.40 tokens per second) +1.23.883.273 I slot print_timing: id 0 | task 104 | total time = 6879.03 ms / 2468 tokens +1.23.883.275 I slot print_timing: id 0 | task 104 | graphs reused = 37 +1.23.883.432 I slot release: id 0 | task 104 | stop processing: n_tokens = 2467, truncated = 0 +1.23.883.451 I srv update_slots: all slots are idle +1.23.929.558 I srv server_strea: conv_id= (empty=1) +1.23.932.081 I srv operator(): chat format: peg-native +1.23.932.290 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +1.23.932.300 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +1.23.932.656 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.23.932.678 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.23.932.683 I slot launch_slot_: id 0 | task 113 | processing task, is_child = 0 +1.23.932.696 I slot operator(): id 0 | task 113 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +1.23.932.699 I slot operator(): id 0 | task 113 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +1.23.932.703 I slot operator(): id 0 | task 113 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.23.933.366 I slot operator(): id 0 | task 113 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.23.933.980 I slot operator(): id 0 | task 113 | cached n_tokens = 0, memory_seq_rm [0, end) +1.23.969.849 I slot operator(): id 0 | task 113 | cached n_tokens = 10, memory_seq_rm [10, end) +1.23.969.954 I slot create_check: id 0 | task 113 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +1.29.218.771 I slot print_timing: id 0 | task 113 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.29 s / 368.33 tokens per second +1.29.218.780 I slot operator(): id 0 | task 113 | cached n_tokens = 1947, memory_seq_rm [1947, end) +1.29.223.125 I slot create_check: id 0 | task 113 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +1.30.687.985 I slot print_timing: id 0 | task 113 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.76 s / 364.01 tokens per second +1.30.687.993 I slot operator(): id 0 | task 113 | cached n_tokens = 2459, memory_seq_rm [2459, end) +1.30.688.324 I slot init_sampler: id 0 | task 113 | init sampler, took 0.30 ms, tokens: text = 2463, total = 2463 +1.30.692.121 I slot create_check: id 0 | task 113 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +1.30.814.911 I slot print_timing: id 0 | task 113 | prompt eval time = 6793.55 ms / 2463 tokens ( 2.76 ms per token, 362.55 tokens per second) +1.30.814.919 I slot print_timing: id 0 | task 113 | eval time = 88.65 ms / 5 tokens ( 22.16 ms per token, 45.12 tokens per second) +1.30.814.920 I slot print_timing: id 0 | task 113 | total time = 6882.20 ms / 2468 tokens +1.30.814.921 I slot print_timing: id 0 | task 113 | graphs reused = 40 +1.30.814.968 I slot release: id 0 | task 113 | stop processing: n_tokens = 2467, truncated = 0 +1.30.814.987 I srv update_slots: all slots are idle +1.30.840.457 I srv server_strea: conv_id= (empty=1) +1.30.843.106 I srv operator(): chat format: peg-native +1.30.843.330 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +1.30.843.341 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +1.30.843.674 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.30.843.699 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.30.843.703 I slot launch_slot_: id 0 | task 122 | processing task, is_child = 0 +1.30.843.714 I slot operator(): id 0 | task 122 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +1.30.843.719 I slot operator(): id 0 | task 122 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +1.30.843.724 I slot operator(): id 0 | task 122 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.30.844.891 I slot operator(): id 0 | task 122 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.30.845.874 I slot operator(): id 0 | task 122 | cached n_tokens = 0, memory_seq_rm [0, end) +1.30.883.537 I slot operator(): id 0 | task 122 | cached n_tokens = 10, memory_seq_rm [10, end) +1.30.883.669 I slot create_check: id 0 | task 122 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +1.36.137.958 I slot print_timing: id 0 | task 122 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.29 s / 367.76 tokens per second +1.36.137.965 I slot operator(): id 0 | task 122 | cached n_tokens = 1947, memory_seq_rm [1947, end) +1.36.142.221 I slot create_check: id 0 | task 122 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +1.37.621.884 I slot print_timing: id 0 | task 122 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.78 s / 362.78 tokens per second +1.37.621.894 I slot operator(): id 0 | task 122 | cached n_tokens = 2459, memory_seq_rm [2459, end) +1.37.622.253 I slot init_sampler: id 0 | task 122 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +1.37.626.533 I slot create_check: id 0 | task 122 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +1.37.745.431 I slot print_timing: id 0 | task 122 | prompt eval time = 6814.12 ms / 2463 tokens ( 2.77 ms per token, 361.46 tokens per second) +1.37.745.440 I slot print_timing: id 0 | task 122 | eval time = 87.58 ms / 5 tokens ( 21.90 ms per token, 45.67 tokens per second) +1.37.745.442 I slot print_timing: id 0 | task 122 | total time = 6901.70 ms / 2468 tokens +1.37.745.442 I slot print_timing: id 0 | task 122 | graphs reused = 43 +1.37.745.488 I slot release: id 0 | task 122 | stop processing: n_tokens = 2467, truncated = 0 +1.37.745.505 I srv update_slots: all slots are idle +1.37.782.280 I srv server_strea: conv_id= (empty=1) +1.37.784.764 I srv operator(): chat format: peg-native +1.37.784.961 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +1.37.784.978 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +1.37.785.316 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.37.785.341 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.37.785.346 I slot launch_slot_: id 0 | task 131 | processing task, is_child = 0 +1.37.785.356 I slot operator(): id 0 | task 131 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +1.37.785.361 I slot operator(): id 0 | task 131 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +1.37.785.365 I slot operator(): id 0 | task 131 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.37.786.207 I slot operator(): id 0 | task 131 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.37.787.006 I slot operator(): id 0 | task 131 | cached n_tokens = 0, memory_seq_rm [0, end) +1.37.824.762 I slot operator(): id 0 | task 131 | cached n_tokens = 10, memory_seq_rm [10, end) +1.37.824.896 I slot create_check: id 0 | task 131 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +1.43.090.102 I slot print_timing: id 0 | task 131 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.30 s / 367.03 tokens per second +1.43.090.109 I slot operator(): id 0 | task 131 | cached n_tokens = 1947, memory_seq_rm [1947, end) +1.43.093.976 I slot create_check: id 0 | task 131 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +1.44.596.668 I slot print_timing: id 0 | task 131 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.81 s / 361.02 tokens per second +1.44.596.676 I slot operator(): id 0 | task 131 | cached n_tokens = 2459, memory_seq_rm [2459, end) +1.44.597.053 I slot init_sampler: id 0 | task 131 | init sampler, took 0.35 ms, tokens: text = 2463, total = 2463 +1.44.601.088 I slot create_check: id 0 | task 131 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +1.44.728.371 I slot print_timing: id 0 | task 131 | prompt eval time = 6848.01 ms / 2463 tokens ( 2.78 ms per token, 359.67 tokens per second) +1.44.728.382 I slot print_timing: id 0 | task 131 | eval time = 94.98 ms / 5 tokens ( 23.75 ms per token, 42.11 tokens per second) +1.44.728.384 I slot print_timing: id 0 | task 131 | total time = 6943.00 ms / 2468 tokens +1.44.728.385 I slot print_timing: id 0 | task 131 | graphs reused = 46 +1.44.728.445 I slot release: id 0 | task 131 | stop processing: n_tokens = 2467, truncated = 0 +1.44.728.463 I srv update_slots: all slots are idle +1.44.755.902 I srv server_strea: conv_id= (empty=1) +1.44.758.275 I srv operator(): chat format: peg-native +1.44.758.470 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +1.44.758.481 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +1.44.758.737 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.44.758.761 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.44.758.765 I slot launch_slot_: id 0 | task 140 | processing task, is_child = 0 +1.44.758.776 I slot operator(): id 0 | task 140 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +1.44.758.781 I slot operator(): id 0 | task 140 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +1.44.758.785 I slot operator(): id 0 | task 140 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.44.759.934 I slot operator(): id 0 | task 140 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.44.760.772 I slot operator(): id 0 | task 140 | cached n_tokens = 0, memory_seq_rm [0, end) +1.44.797.388 I slot operator(): id 0 | task 140 | cached n_tokens = 10, memory_seq_rm [10, end) +1.44.797.553 I slot create_check: id 0 | task 140 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +1.50.081.483 I slot print_timing: id 0 | task 140 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.32 s / 365.79 tokens per second +1.50.081.490 I slot operator(): id 0 | task 140 | cached n_tokens = 1947, memory_seq_rm [1947, end) +1.50.085.299 I slot create_check: id 0 | task 140 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +1.51.578.259 I slot print_timing: id 0 | task 140 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.82 s / 360.59 tokens per second +1.51.578.266 I slot operator(): id 0 | task 140 | cached n_tokens = 2459, memory_seq_rm [2459, end) +1.51.578.564 I slot init_sampler: id 0 | task 140 | init sampler, took 0.28 ms, tokens: text = 2463, total = 2463 +1.51.582.332 I slot create_check: id 0 | task 140 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +1.51.708.472 I slot print_timing: id 0 | task 140 | prompt eval time = 6856.15 ms / 2463 tokens ( 2.78 ms per token, 359.24 tokens per second) +1.51.708.484 I slot print_timing: id 0 | task 140 | eval time = 93.53 ms / 5 tokens ( 23.38 ms per token, 42.77 tokens per second) +1.51.708.486 I slot print_timing: id 0 | task 140 | total time = 6949.68 ms / 2468 tokens +1.51.708.487 I slot print_timing: id 0 | task 140 | graphs reused = 49 +1.51.708.546 I slot release: id 0 | task 140 | stop processing: n_tokens = 2467, truncated = 0 +1.51.708.565 I srv update_slots: all slots are idle +1.51.745.471 I srv server_strea: conv_id= (empty=1) +1.51.748.103 I srv operator(): chat format: peg-native +1.51.748.320 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +1.51.748.327 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +1.51.748.592 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.51.748.607 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.51.748.611 I slot launch_slot_: id 0 | task 149 | processing task, is_child = 0 +1.51.748.622 I slot operator(): id 0 | task 149 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +1.51.748.626 I slot operator(): id 0 | task 149 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +1.51.748.628 I slot operator(): id 0 | task 149 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.51.749.370 I slot operator(): id 0 | task 149 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.51.750.076 I slot operator(): id 0 | task 149 | cached n_tokens = 0, memory_seq_rm [0, end) +1.51.789.740 I slot operator(): id 0 | task 149 | cached n_tokens = 10, memory_seq_rm [10, end) +1.51.789.843 I slot create_check: id 0 | task 149 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +1.57.080.221 I slot print_timing: id 0 | task 149 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.33 s / 365.18 tokens per second +1.57.080.228 I slot operator(): id 0 | task 149 | cached n_tokens = 1947, memory_seq_rm [1947, end) +1.57.084.328 I slot create_check: id 0 | task 149 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +1.58.563.904 I slot print_timing: id 0 | task 149 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.82 s / 360.81 tokens per second +1.58.563.914 I slot operator(): id 0 | task 149 | cached n_tokens = 2459, memory_seq_rm [2459, end) +1.58.564.277 I slot init_sampler: id 0 | task 149 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +1.58.568.306 I slot create_check: id 0 | task 149 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +1.58.689.341 I slot print_timing: id 0 | task 149 | prompt eval time = 6851.83 ms / 2463 tokens ( 2.78 ms per token, 359.47 tokens per second) +1.58.689.356 I slot print_timing: id 0 | task 149 | eval time = 88.87 ms / 5 tokens ( 22.22 ms per token, 45.01 tokens per second) +1.58.689.358 I slot print_timing: id 0 | task 149 | total time = 6940.70 ms / 2468 tokens +1.58.689.359 I slot print_timing: id 0 | task 149 | graphs reused = 52 +1.58.689.422 I slot release: id 0 | task 149 | stop processing: n_tokens = 2467, truncated = 0 +1.58.689.445 I srv update_slots: all slots are idle +1.58.726.886 I srv server_strea: conv_id= (empty=1) +1.58.729.071 I srv operator(): chat format: peg-native +1.58.729.232 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +1.58.729.249 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +1.58.729.571 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +1.58.729.595 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +1.58.729.598 I slot launch_slot_: id 0 | task 158 | processing task, is_child = 0 +1.58.729.609 I slot operator(): id 0 | task 158 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +1.58.729.614 I slot operator(): id 0 | task 158 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +1.58.729.618 I slot operator(): id 0 | task 158 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.58.730.766 I slot operator(): id 0 | task 158 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +1.58.731.574 I slot operator(): id 0 | task 158 | cached n_tokens = 0, memory_seq_rm [0, end) +1.58.766.683 I slot operator(): id 0 | task 158 | cached n_tokens = 10, memory_seq_rm [10, end) +1.58.766.848 I slot create_check: id 0 | task 158 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +2.03.950.027 I slot print_timing: id 0 | task 158 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.22 s / 372.96 tokens per second +2.03.950.035 I slot operator(): id 0 | task 158 | cached n_tokens = 1947, memory_seq_rm [1947, end) +2.03.954.013 I slot create_check: id 0 | task 158 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +2.05.430.141 I slot print_timing: id 0 | task 158 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.70 s / 366.99 tokens per second +2.05.430.147 I slot operator(): id 0 | task 158 | cached n_tokens = 2459, memory_seq_rm [2459, end) +2.05.430.507 I slot init_sampler: id 0 | task 158 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +2.05.434.442 I slot create_check: id 0 | task 158 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +2.05.560.628 I slot print_timing: id 0 | task 158 | prompt eval time = 6738.13 ms / 2463 tokens ( 2.74 ms per token, 365.53 tokens per second) +2.05.560.642 I slot print_timing: id 0 | task 158 | eval time = 92.87 ms / 5 tokens ( 23.22 ms per token, 43.07 tokens per second) +2.05.560.644 I slot print_timing: id 0 | task 158 | total time = 6831.00 ms / 2468 tokens +2.05.560.645 I slot print_timing: id 0 | task 158 | graphs reused = 55 +2.05.560.707 I slot release: id 0 | task 158 | stop processing: n_tokens = 2467, truncated = 0 +2.05.560.728 I srv update_slots: all slots are idle +2.05.597.271 I srv server_strea: conv_id= (empty=1) +2.05.600.312 I srv operator(): chat format: peg-native +2.05.600.506 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +2.05.600.516 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +2.05.600.760 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +2.05.600.776 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +2.05.600.778 I slot launch_slot_: id 0 | task 167 | processing task, is_child = 0 +2.05.600.786 I slot operator(): id 0 | task 167 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +2.05.600.790 I slot operator(): id 0 | task 167 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +2.05.600.793 I slot operator(): id 0 | task 167 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +2.05.601.627 I slot operator(): id 0 | task 167 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +2.05.602.445 I slot operator(): id 0 | task 167 | cached n_tokens = 0, memory_seq_rm [0, end) +2.05.640.893 I slot operator(): id 0 | task 167 | cached n_tokens = 10, memory_seq_rm [10, end) +2.05.640.996 I slot create_check: id 0 | task 167 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +2.10.807.376 I slot print_timing: id 0 | task 167 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.21 s / 373.95 tokens per second +2.10.807.394 I slot operator(): id 0 | task 167 | cached n_tokens = 1947, memory_seq_rm [1947, end) +2.10.811.684 I slot create_check: id 0 | task 167 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +2.12.281.758 I slot print_timing: id 0 | task 167 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.68 s / 368.06 tokens per second +2.12.281.764 I slot operator(): id 0 | task 167 | cached n_tokens = 2459, memory_seq_rm [2459, end) +2.12.282.033 I slot init_sampler: id 0 | task 167 | init sampler, took 0.25 ms, tokens: text = 2463, total = 2463 +2.12.285.639 I slot create_check: id 0 | task 167 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +2.12.407.964 I slot print_timing: id 0 | task 167 | prompt eval time = 6717.38 ms / 2463 tokens ( 2.73 ms per token, 366.66 tokens per second) +2.12.407.980 I slot print_timing: id 0 | task 167 | eval time = 89.77 ms / 5 tokens ( 22.44 ms per token, 44.56 tokens per second) +2.12.407.983 I slot print_timing: id 0 | task 167 | total time = 6807.16 ms / 2468 tokens +2.12.407.985 I slot print_timing: id 0 | task 167 | graphs reused = 58 +2.12.408.090 I slot release: id 0 | task 167 | stop processing: n_tokens = 2467, truncated = 0 +2.12.408.117 I srv update_slots: all slots are idle +2.12.456.506 I srv server_strea: conv_id= (empty=1) +2.12.459.497 I srv operator(): chat format: peg-native +2.12.459.704 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +2.12.459.712 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +2.12.459.957 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +2.12.459.970 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +2.12.459.972 I slot launch_slot_: id 0 | task 176 | processing task, is_child = 0 +2.12.459.980 I slot operator(): id 0 | task 176 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +2.12.459.983 I slot operator(): id 0 | task 176 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +2.12.459.986 I slot operator(): id 0 | task 176 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +2.12.460.807 I slot operator(): id 0 | task 176 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +2.12.461.639 I slot operator(): id 0 | task 176 | cached n_tokens = 0, memory_seq_rm [0, end) +2.12.498.419 I slot operator(): id 0 | task 176 | cached n_tokens = 10, memory_seq_rm [10, end) +2.12.498.546 I slot create_check: id 0 | task 176 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +2.17.635.321 I slot print_timing: id 0 | task 176 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.18 s / 376.21 tokens per second +2.17.635.337 I slot operator(): id 0 | task 176 | cached n_tokens = 1947, memory_seq_rm [1947, end) +2.17.639.507 I slot create_check: id 0 | task 176 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +2.19.095.914 I slot print_timing: id 0 | task 176 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.64 s / 370.56 tokens per second +2.19.095.926 I slot operator(): id 0 | task 176 | cached n_tokens = 2459, memory_seq_rm [2459, end) +2.19.096.289 I slot init_sampler: id 0 | task 176 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +2.19.100.249 I slot create_check: id 0 | task 176 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +2.19.222.721 I slot print_timing: id 0 | task 176 | prompt eval time = 6672.30 ms / 2463 tokens ( 2.71 ms per token, 369.14 tokens per second) +2.19.222.734 I slot print_timing: id 0 | task 176 | eval time = 90.43 ms / 5 tokens ( 22.61 ms per token, 44.24 tokens per second) +2.19.222.735 I slot print_timing: id 0 | task 176 | total time = 6762.72 ms / 2468 tokens +2.19.222.737 I slot print_timing: id 0 | task 176 | graphs reused = 61 +2.19.222.797 I slot release: id 0 | task 176 | stop processing: n_tokens = 2467, truncated = 0 +2.19.222.817 I srv update_slots: all slots are idle +2.19.246.460 I srv server_strea: conv_id= (empty=1) +2.19.249.844 I srv operator(): chat format: peg-native +2.19.250.082 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +2.19.250.092 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +2.19.250.348 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +2.19.250.365 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +2.19.250.368 I slot launch_slot_: id 0 | task 185 | processing task, is_child = 0 +2.19.250.376 I slot operator(): id 0 | task 185 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +2.19.250.380 I slot operator(): id 0 | task 185 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +2.19.250.383 I slot operator(): id 0 | task 185 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +2.19.251.274 I slot operator(): id 0 | task 185 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +2.19.252.175 I slot operator(): id 0 | task 185 | cached n_tokens = 0, memory_seq_rm [0, end) +2.19.289.367 I slot operator(): id 0 | task 185 | cached n_tokens = 10, memory_seq_rm [10, end) +2.19.289.495 I slot create_check: id 0 | task 185 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +2.24.460.590 I slot print_timing: id 0 | task 185 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.21 s / 373.69 tokens per second +2.24.460.600 I slot operator(): id 0 | task 185 | cached n_tokens = 1947, memory_seq_rm [1947, end) +2.24.464.683 I slot create_check: id 0 | task 185 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +2.25.966.474 I slot print_timing: id 0 | task 185 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.72 s / 366.14 tokens per second +2.25.966.481 I slot operator(): id 0 | task 185 | cached n_tokens = 2459, memory_seq_rm [2459, end) +2.25.966.763 I slot init_sampler: id 0 | task 185 | init sampler, took 0.26 ms, tokens: text = 2463, total = 2463 +2.25.970.350 I slot create_check: id 0 | task 185 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +2.26.092.547 I slot print_timing: id 0 | task 185 | prompt eval time = 6751.77 ms / 2463 tokens ( 2.74 ms per token, 364.79 tokens per second) +2.26.092.557 I slot print_timing: id 0 | task 185 | eval time = 90.38 ms / 5 tokens ( 22.60 ms per token, 44.26 tokens per second) +2.26.092.559 I slot print_timing: id 0 | task 185 | total time = 6842.15 ms / 2468 tokens +2.26.092.560 I slot print_timing: id 0 | task 185 | graphs reused = 64 +2.26.092.621 I slot release: id 0 | task 185 | stop processing: n_tokens = 2467, truncated = 0 +2.26.092.640 I srv update_slots: all slots are idle +2.26.162.578 I srv server_strea: conv_id= (empty=1) +2.26.165.868 I srv operator(): chat format: peg-native +2.26.166.098 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +2.26.166.106 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +2.26.166.325 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +2.26.166.339 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +2.26.166.341 I slot launch_slot_: id 0 | task 194 | processing task, is_child = 0 +2.26.166.350 I slot operator(): id 0 | task 194 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +2.26.166.354 I slot operator(): id 0 | task 194 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +2.26.166.357 I slot operator(): id 0 | task 194 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +2.26.167.058 I slot operator(): id 0 | task 194 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +2.26.167.809 I slot operator(): id 0 | task 194 | cached n_tokens = 0, memory_seq_rm [0, end) +2.26.203.485 I slot operator(): id 0 | task 194 | cached n_tokens = 10, memory_seq_rm [10, end) +2.26.203.608 I slot create_check: id 0 | task 194 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +2.31.448.023 I slot print_timing: id 0 | task 194 | prompt processing, n_tokens = 1947, progress = 0.79, t = 5.28 s / 368.64 tokens per second +2.31.448.034 I slot operator(): id 0 | task 194 | cached n_tokens = 1947, memory_seq_rm [1947, end) +2.31.452.141 I slot create_check: id 0 | task 194 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +2.32.938.387 I slot print_timing: id 0 | task 194 | prompt processing, n_tokens = 2459, progress = 1.00, t = 6.77 s / 363.11 tokens per second +2.32.938.399 I slot operator(): id 0 | task 194 | cached n_tokens = 2459, memory_seq_rm [2459, end) +2.32.938.762 I slot init_sampler: id 0 | task 194 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +2.32.943.138 I slot create_check: id 0 | task 194 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +2.33.067.483 I slot print_timing: id 0 | task 194 | prompt eval time = 6809.65 ms / 2463 tokens ( 2.76 ms per token, 361.69 tokens per second) +2.33.067.498 I slot print_timing: id 0 | task 194 | eval time = 91.47 ms / 5 tokens ( 22.87 ms per token, 43.73 tokens per second) +2.33.067.500 I slot print_timing: id 0 | task 194 | total time = 6901.11 ms / 2468 tokens +2.33.067.501 I slot print_timing: id 0 | task 194 | graphs reused = 67 +2.33.067.561 I slot release: id 0 | task 194 | stop processing: n_tokens = 2467, truncated = 0 +2.33.067.582 I srv update_slots: all slots are idle diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_final_smoke_20260923.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_final_smoke_20260923.driver.log new file mode 100644 index 000000000..4525d17f0 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_final_smoke_20260923.driver.log @@ -0,0 +1,171 @@ +INFO 09-23 20:28:05 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 20:28:06 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 20:28:09 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +WARNING 09-23 20:28:09 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 13] Permission denied: 'C:\\Users\\zhout\\w2' +{ + "scope": "complete Spark GGUF text requests through Omni StageRuntime/StagePool and external llama.cpp backend", + "model_file": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "device": "Vulkan1", + "memory_budget": { + "capacities": { + "host_ram": 12884901888 + }, + "demands": { + "host_ram": 4294967296 + } + }, + "warmup_count": 0, + "measured_count": 1, + "startup_s": 3.379536999971606, + "execution_plan": { + "backend": "external.llamacpp.text.v1", + "stage_id": 0, + "worker_generation": "35c3138d13684fedaa00c41b466777de", + "model_sha256": "902bde2522394954ac17821b3e5fd0df02defbc6944f122253f2580acf0503f4", + "server_sha256": "9ffc5919acb4cb43c7be5f3053b59014cce70a87d3a801fcad49c4f459984f52", + "model_alias": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "requested_device": "Vulkan1", + "expected_device_name": "AMD Radeon(TM) 890M Graphics", + "layer_assignments": 0, + "offloaded_layers": [ + "29", + "29" + ], + "model_buffer_devices": [ + "Vulkan1" + ], + "worker_pid": 44008, + "loaded_rss_bytes": 1640878080, + "reserved_bytes": { + "host_ram": 4294967296 + }, + "memory_pool": "host_ram", + "memory_overhead_bytes": 2147483648, + "context_tokens": 4096, + "max_new_tokens": 96, + "max_io_bytes": 1048576, + "request_capacity": 1, + "stateful_session": "llama.cpp-owned; reset for each complete request", + "evidence": "B" + }, + "checks": [ + { + "case": "short", + "answer": "Paris", + "expected": "Paris", + "correct": true, + "wall_s": 0.1326008999021724, + "stage_event": { + "request_id": "short-621227093000000", + "stage_id": 0, + "epoch": 1, + "seq": 1, + "kind": "text", + "worker_generation": "35c3138d13684fedaa00c41b466777de", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 0.1288311000680551, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 28, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.094885600032285, + "stage_event": { + "request_id": "inventory_120-621227218000000", + "stage_id": 0, + "epoch": 2, + "seq": 1, + "kind": "text", + "worker_generation": "35c3138d13684fedaa00c41b466777de", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.0779674999648705, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "nearest_rank_p50_wall_s": 2.0904675000347197, + "nearest_rank_p95_wall_s": 2.0904675000347197, + "context_check": { + "rejected": true, + "error": "ResourceUnavailable: Spark prompt plus generation bound exceeds context; refusing truncation", + "recovery": { + "case": "short", + "answer": "Paris", + "expected": "Paris", + "correct": true, + "wall_s": 0.1633006000192836, + "stage_event": { + "request_id": "short-621231437000000", + "stage_id": 0, + "epoch": 4, + "seq": 1, + "kind": "text", + "worker_generation": "35c3138d13684fedaa00c41b466777de", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 0.14913889998570085, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 28, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + }, + "abort_check": { + "stale_output": false, + "worker_exited": true, + "ledger_after_abort": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } + }, + "status": "passed", + "ledger_after_shutdown": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_final_smoke_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_final_smoke_20260923.json new file mode 100644 index 000000000..2a1329d59 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_final_smoke_20260923.json @@ -0,0 +1,199 @@ +{ + "scope": "complete Spark GGUF text requests through Omni StageRuntime/StagePool and external llama.cpp backend", + "model_file": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "device": "Vulkan1", + "memory_budget": { + "capacities": { + "host_ram": 12884901888 + }, + "demands": { + "host_ram": 4294967296 + } + }, + "warmup_count": 0, + "measured_count": 1, + "startup_s": 3.379536999971606, + "execution_plan": { + "backend": "external.llamacpp.text.v1", + "stage_id": 0, + "worker_generation": "35c3138d13684fedaa00c41b466777de", + "model_sha256": "902bde2522394954ac17821b3e5fd0df02defbc6944f122253f2580acf0503f4", + "server_sha256": "9ffc5919acb4cb43c7be5f3053b59014cce70a87d3a801fcad49c4f459984f52", + "model_alias": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "requested_device": "Vulkan1", + "expected_device_name": "AMD Radeon(TM) 890M Graphics", + "layer_assignments": 0, + "offloaded_layers": [ + "29", + "29" + ], + "model_buffer_devices": [ + "Vulkan1" + ], + "worker_pid": 44008, + "loaded_rss_bytes": 1640878080, + "reserved_bytes": { + "host_ram": 4294967296 + }, + "memory_pool": "host_ram", + "memory_overhead_bytes": 2147483648, + "context_tokens": 4096, + "max_new_tokens": 96, + "max_io_bytes": 1048576, + "request_capacity": 1, + "stateful_session": "llama.cpp-owned; reset for each complete request", + "evidence": "B" + }, + "checks": [ + { + "case": "short", + "answer": "Paris", + "expected": "Paris", + "correct": true, + "wall_s": 0.1326008999021724, + "stage_event": { + "request_id": "short-621227093000000", + "stage_id": 0, + "epoch": 1, + "seq": 1, + "kind": "text", + "worker_generation": "35c3138d13684fedaa00c41b466777de", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 0.1288311000680551, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 28, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.094885600032285, + "stage_event": { + "request_id": "inventory_120-621227218000000", + "stage_id": 0, + "epoch": 2, + "seq": 1, + "kind": "text", + "worker_generation": "35c3138d13684fedaa00c41b466777de", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.0779674999648705, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "warmups": [], + "measured": [ + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.0904675000347197, + "stage_event": { + "request_id": "inventory_120-621229312000000", + "stage_id": 0, + "epoch": 3, + "seq": 1, + "kind": "text", + "worker_generation": "35c3138d13684fedaa00c41b466777de", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.059911199961789, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "nearest_rank_p50_wall_s": 2.0904675000347197, + "nearest_rank_p95_wall_s": 2.0904675000347197, + "context_check": { + "rejected": true, + "error": "ResourceUnavailable: Spark prompt plus generation bound exceeds context; refusing truncation", + "recovery": { + "case": "short", + "answer": "Paris", + "expected": "Paris", + "correct": true, + "wall_s": 0.1633006000192836, + "stage_event": { + "request_id": "short-621231437000000", + "stage_id": 0, + "epoch": 4, + "seq": 1, + "kind": "text", + "worker_generation": "35c3138d13684fedaa00c41b466777de", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 0.14913889998570085, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 28, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + }, + "abort_check": { + "stale_output": false, + "worker_exited": true, + "ledger_after_abort": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } + }, + "status": "passed", + "ledger_after_shutdown": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_final_smoke_20260923.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_final_smoke_20260923.log new file mode 100644 index 000000000..badf32a32 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_final_smoke_20260923.log @@ -0,0 +1,299 @@ +0.00.000.703 I srv llama_server: initializing ... +0.00.429.983 I cmn common_param: common_params_print_info: build 11124 (1a679828f) with Clang 20.1.8 for Windows x86_64 +0.00.429.988 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg) +0.00.429.990 I cmn common_param: device_info: +0.00.433.185 I cmn common_param: - Vulkan0 : NVIDIA GeForce RTX 5090 Laptop GPU (24137 MiB, 23369 MiB free) +0.00.436.250 I cmn common_param: - Vulkan1 : AMD Radeon(TM) 890M Graphics (48505 MiB, 46080 MiB free) +0.00.436.257 I cmn common_param: - CPU : AMD Ryzen AI 9 HX 370 w/ Radeon 890M (64633 MiB, 30664 MiB free) +0.00.436.307 I cmn common_param: system_info: n_threads = 12 (n_threads_batch = 12) / 24 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.436.379 I srv init_listene: running without SSL +0.00.436.433 I srv init_listene: The UI is disabled +0.00.436.433 I srv init_listene: Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable +0.00.436.437 I srv init: using 23 threads for HTTP server +0.00.436.496 W srv llama_server: security: no API key is set and CORS allows all origins (see https://github.com/ggml-org/llama.cpp/pull/25655) +0.00.444.621 I srv load_model: loading model 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' +0.00.444.632 I srv load_model: local path 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' +0.00.468.881 I llama_model_loader: loaded meta data with 37 key-value pairs and 226 tensors from C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf (version GGUF V3 (latest)) +0.00.468.895 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +0.00.468.902 I llama_model_loader: - kv 0: general.architecture str = spark2_5 +0.00.468.902 I llama_model_loader: - kv 1: general.type str = model +0.00.468.906 I llama_model_loader: - kv 2: general.sampling.top_k i32 = 0 +0.00.468.913 I llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +0.00.468.915 I llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +0.00.468.918 I llama_model_loader: - kv 5: general.name str = Hf_Format +0.00.468.918 I llama_model_loader: - kv 6: general.size_label str = 1.7B +0.00.468.921 I llama_model_loader: - kv 7: spark2_5.block_count u32 = 28 +0.00.468.922 I llama_model_loader: - kv 8: spark2_5.context_length u32 = 1048576 +0.00.468.922 I llama_model_loader: - kv 9: spark2_5.embedding_length u32 = 2048 +0.00.468.922 I llama_model_loader: - kv 10: spark2_5.feed_forward_length u32 = 6656 +0.00.468.923 I llama_model_loader: - kv 11: spark2_5.attention.head_count u32 = 8 +0.00.468.923 I llama_model_loader: - kv 12: spark2_5.attention.head_count_kv u32 = 2 +0.00.468.924 I llama_model_loader: - kv 13: spark2_5.rope.freq_base f32 = 5000000.000000 +0.00.468.925 I llama_model_loader: - kv 14: spark2_5.rope.freq_base_swa f32 = 10000.000000 +0.00.468.927 I llama_model_loader: - kv 15: spark2_5.attention.layer_norm_rms_epsilon f32 = 0.000001 +0.00.468.928 I llama_model_loader: - kv 16: spark2_5.attention.key_length u32 = 256 +0.00.468.928 I llama_model_loader: - kv 17: spark2_5.attention.value_length u32 = 256 +0.00.468.928 I llama_model_loader: - kv 18: spark2_5.vocab_size u32 = 131072 +0.00.468.929 I llama_model_loader: - kv 19: spark2_5.attention.sliding_window u32 = 512 +0.00.468.949 I llama_model_loader: - kv 20: spark2_5.attention.sliding_window_pattern arr[bool,28] = [true, true, true, false, true, true,... +0.00.468.950 I llama_model_loader: - kv 21: spark2_5.rope.dimension_count u32 = 64 +0.00.468.950 I llama_model_loader: - kv 22: spark2_5.rope.dimension_count_swa u32 = 256 +0.00.468.951 I llama_model_loader: - kv 23: tokenizer.ggml.model str = gpt2 +0.00.468.951 I llama_model_loader: - kv 24: tokenizer.ggml.pre str = spark2_5 +0.00.484.432 I llama_model_loader: - kv 25: tokenizer.ggml.tokens arr[str,131072] = ["<｜start▁of▁sentence｜>", "<... +0.00.489.120 I llama_model_loader: - kv 26: tokenizer.ggml.token_type arr[i32,131072] = [3, 3, 3, 4, 4, 3, 3, 3, 4, 4, 4, 3, ... +0.00.504.479 I llama_model_loader: - kv 27: tokenizer.ggml.merges arr[str,130716] = ["Ġ Ġ", "Ġ t", "i n", "Ġ a", "Ġ... +0.00.504.485 I llama_model_loader: - kv 28: tokenizer.ggml.bos_token_id u32 = 0 +0.00.504.486 I llama_model_loader: - kv 29: tokenizer.ggml.eos_token_id u32 = 1 +0.00.504.486 I llama_model_loader: - kv 30: tokenizer.ggml.unknown_token_id u32 = 5 +0.00.504.487 I llama_model_loader: - kv 31: tokenizer.ggml.padding_token_id u32 = 2 +0.00.504.487 I llama_model_loader: - kv 32: tokenizer.ggml.add_bos_token bool = false +0.00.504.488 I llama_model_loader: - kv 33: tokenizer.ggml.add_eos_token bool = false +0.00.504.491 I llama_model_loader: - kv 34: tokenizer.chat_template str = {%- if not messages %}\n {{- raise_... +0.00.504.492 I llama_model_loader: - kv 35: general.quantization_version u32 = 2 +0.00.504.493 I llama_model_loader: - kv 36: general.file_type u32 = 15 +0.00.504.499 I llama_model_loader: - type f32: 57 tensors +0.00.504.500 I llama_model_loader: - type q4_K: 140 tensors +0.00.504.500 I llama_model_loader: - type q6_K: 29 tensors +0.00.504.505 I print_info: file format = GGUF V3 (latest) +0.00.504.505 I print_info: file type = Q4_K - Medium +0.00.504.512 I print_info: file size = 1.03 GiB (5.16 BPW) +0.00.509.206 I llama_prepare_model_devices: using device Vulkan1 (AMD Radeon(TM) 890M Graphics) (unknown id) - 46080 MiB free +0.00.560.208 I load: 86 unused tokens +0.00.575.115 I load: printing all EOG tokens: +0.00.575.121 I load: - 1 ('<｜end▁of▁sentence｜>') +0.00.575.256 I load: special tokens cache size = 113 +0.00.591.350 I load: token to piece cache size = 0.7975 MB +0.00.591.385 I print_info: arch = spark2_5 +0.00.591.386 I print_info: vocab_only = 0 +0.00.591.387 I print_info: no_alloc = 0 +0.00.591.387 I print_info: n_ctx_train = 1048576 +0.00.591.388 I print_info: n_embd_inp = 2048 +0.00.591.388 I print_info: n_embd = 2048 +0.00.591.389 I print_info: n_embd_out = 2048 +0.00.591.389 I print_info: n_layer = 28 +0.00.591.389 I print_info: n_layer_all = 28 +0.00.591.409 I print_info: n_head = 8 +0.00.591.410 I print_info: n_head_kv = 2 +0.00.591.411 I print_info: n_rot = 64 +0.00.591.411 I print_info: n_swa = 512 +0.00.591.411 I print_info: is_swa_any = 1 +0.00.591.411 I print_info: non_causal_type = 0 +0.00.591.412 I print_info: n_embd_head_k = 256 +0.00.591.412 I print_info: n_embd_head_v = 256 +0.00.591.413 I print_info: n_gqa = 4 +0.00.591.415 I print_info: n_embd_k_gqa = 512 +0.00.591.416 I print_info: n_embd_v_gqa = 512 +0.00.591.418 I print_info: f_norm_eps = 0.0e+00 +0.00.591.419 I print_info: f_norm_rms_eps = 1.0e-06 +0.00.591.419 I print_info: f_clamp_kqv = 0.0e+00 +0.00.591.420 I print_info: f_max_alibi_bias = 0.0e+00 +0.00.591.420 I print_info: f_logit_scale = 0.0e+00 +0.00.591.420 I print_info: f_attn_scale = 0.0e+00 +0.00.591.421 I print_info: f_attn_value_scale = 0.0000 +0.00.591.422 I print_info: n_ff = 6656 +0.00.591.423 I print_info: n_expert = 0 +0.00.591.423 I print_info: n_expert_used = 0 +0.00.591.424 I print_info: n_expert_groups = 0 +0.00.591.425 I print_info: n_group_used = 0 +0.00.591.425 I print_info: causal attn = 1 +0.00.591.425 I print_info: pooling type = -1 +0.00.591.426 I print_info: rope type = 2 +0.00.591.426 I print_info: rope scaling = linear +0.00.591.427 I print_info: freq_base_train = 5000000.0 +0.00.591.429 I print_info: freq_scale_train = 1 +0.00.591.429 I print_info: freq_base_swa = 10000.0 +0.00.591.430 I print_info: freq_scale_swa = 1 +0.00.591.430 I print_info: n_embd_head_k_swa = 256 +0.00.591.430 I print_info: n_embd_head_v_swa = 256 +0.00.591.431 I print_info: n_rot_swa = 256 +0.00.591.431 I print_info: n_ctx_orig_yarn = 1048576 +0.00.591.431 I print_info: rope_yarn_log_mul = 0.0000 +0.00.591.432 I print_info: rope_finetuned = unknown +0.00.591.433 I print_info: model type = 1.7B +0.00.591.433 I print_info: model params = 1.71 B +0.00.591.434 I print_info: general.name = Hf_Format +0.00.591.435 I print_info: vocab type = BPE +0.00.591.435 I print_info: n_vocab = 131072 +0.00.591.435 I print_info: n_merges = 130716 +0.00.591.436 I print_info: BOS token = 0 '<｜start▁of▁sentence｜>' +0.00.591.437 I print_info: EOS token = 1 '<｜end▁of▁sentence｜>' +0.00.591.437 I print_info: EOT token = 1 '<｜end▁of▁sentence｜>' +0.00.591.438 I print_info: UNK token = 5 '' +0.00.591.438 I print_info: PAD token = 2 '<｜▁pad▁｜>' +0.00.591.439 I print_info: LF token = 198 'Ċ' +0.00.591.439 I print_info: FIM PRE token = 11 '<｜fim▁begin｜>' +0.00.591.440 I print_info: FIM SUF token = 12 '<｜fim▁hole｜>' +0.00.591.440 I print_info: FIM MID token = 13 '<｜fim▁end｜>' +0.00.591.441 I print_info: EOG token = 1 '<｜end▁of▁sentence｜>' +0.00.591.441 I print_info: max token length = 210 +0.00.598.004 I load_tensors: loading model tensors, this can take a while... (load_mode = none) +0.01.195.342 I load_tensors: offloading output layer to GPU +0.01.195.348 I load_tensors: offloading 27 repeating layers to GPU +0.01.195.348 I load_tensors: offloaded 29/29 layers to GPU +0.01.195.353 I load_tensors: Vulkan1 model buffer size = 1051.02 MiB +0.01.195.354 I load_tensors: Vulkan_Host model buffer size = 210.00 MiB +0.01.551.247 I cmn common_init_: added <｜end▁of▁sentence｜> logit bias = -inf +0.01.551.588 I llama_context: constructing llama_context +0.01.551.594 I llama_context: n_seq_max = 1 +0.01.551.595 I llama_context: n_ctx = 4096 +0.01.551.595 I llama_context: n_ctx_seq = 4096 +0.01.551.596 I llama_context: n_batch = 2048 +0.01.551.596 I llama_context: n_ubatch = 512 +0.01.551.597 I llama_context: causal_attn = 1 +0.01.551.597 I llama_context: flash_attn = auto +0.01.551.598 I llama_context: kv_unified = false +0.01.551.601 I llama_context: freq_base = 5000000.0 +0.01.551.603 I llama_context: freq_scale = 1 +0.01.551.603 I llama_context: n_rs_seq = 0 +0.01.551.604 I llama_context: n_outputs_max = 1 +0.01.551.604 I llama_context: n_outputs_max_per_seq = 1 +0.01.551.605 I llama_context: n_ctx_seq (4096) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized +0.01.551.906 I llama_context: Vulkan_Host output buffer size = 0.50 MiB +0.01.551.913 I llama_kv_cache_iswa: creating non-SWA KV cache, size = 4096 cells +0.01.556.514 I llama_kv_cache: Vulkan1 KV buffer size = 56.00 MiB +0.01.557.057 I llama_kv_cache: size = 56.00 MiB ( 4096 cells, 7 layers, 1/1 seqs), K (f16): 28.00 MiB, V (f16): 28.00 MiB +0.01.557.065 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.01.557.066 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.01.557.070 I llama_kv_cache_iswa: creating SWA KV cache, size = 1024 cells +0.01.560.520 I llama_kv_cache: Vulkan1 KV buffer size = 42.00 MiB +0.01.560.999 I llama_kv_cache: size = 42.00 MiB ( 1024 cells, 21 layers, 1/1 seqs), K (f16): 21.00 MiB, V (f16): 21.00 MiB +0.01.561.006 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.01.561.007 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.01.561.014 I sched_reserve: reserving ... +0.01.562.450 I resolve_fused_ops: Flash Attention enabled +0.01.562.452 I resolve_fused_ops: resolving fused DeepSeek V4 HC support: +0.01.563.333 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled +0.01.564.247 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled +0.01.565.293 I resolve_fused_ops: fused DeepSeek V4 HC post enabled +0.01.572.983 I sched_reserve: Vulkan1 compute buffer size = 56.02 MiB +0.01.572.988 I sched_reserve: Vulkan_Host compute buffer size = 13.02 MiB +0.01.572.989 I sched_reserve: graph: nodes = 986, splits = 2, input objects = 4, input tensors = 9 +0.01.572.990 I sched_reserve: reserve took 11.97 ms, sched copies = 1 +0.01.573.093 I cmn init: llama threadpool init, n_threads = 12 +0.01.573.107 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.01.621.729 I srv load_model: initializing, n_slots = 1, n_ctx_slot = 4096, kv_unified = 'false' +0.01.621.750 I spec common_specu: no implementations specified for speculative decoding +0.01.621.752 I slot load_model: id 0 | task -1 | new slot, n_ctx = 4096 +0.01.621.843 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it +0.01.621.845 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.01.621.846 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192 +0.01.621.868 W srv init: --cache-idle-slots requires --cache-ram, disabling +0.01.631.597 I srv init: init: chat template, example_format: '<｜start▁of▁sentence｜><|System|> +you are a helpful assistant. + +You are a helpful assistant<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>Hello<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>Hi there<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>How are you?<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>' +0.01.638.393 I srv init: init: chat template, thinking = 0 +0.01.638.401 I srv init: preserve_reasoning kwarg: true +0.01.638.402 W srv init: chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve) +0.01.638.420 I srv llama_server: model loaded +0.01.638.423 I srv llama_server: listening on http://127.0.0.1:64963 +0.01.638.481 I srv update_slots: all slots are idle +0.01.851.062 I srv server_strea: conv_id= (empty=1) +0.01.851.259 I srv operator(): chat format: peg-native +0.01.851.392 I slot get_availabl: id 0 | task -1 | - skipping, slot is empty +0.01.851.405 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.01.851.473 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.01.851.506 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.01.851.515 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.01.851.533 I slot operator(): id 0 | task 0 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 28 +0.01.851.542 I slot operator(): id 0 | task 0 | cached n_tokens = 0, memory_seq_rm [0, end) +0.01.861.620 I slot operator(): id 0 | task 0 | cached n_tokens = 10, memory_seq_rm [10, end) +0.01.893.574 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.01.896.270 I slot operator(): id 0 | task 0 | cached n_tokens = 24, memory_seq_rm [24, end) +0.01.896.293 I slot init_sampler: id 0 | task 0 | init sampler, took 0.00 ms, tokens: text = 28, total = 28 +0.01.931.963 I slot create_check: id 0 | task 0 | created context checkpoint 2 of 32 (pos_min = 0, pos_max = 23, n_tokens = 24, size = 0.985 MiB) +0.01.969.737 I slot print_timing: id 0 | task 0 | prompt eval time = 101.18 ms / 28 tokens ( 3.61 ms per token, 276.73 tokens per second) +0.01.969.743 I slot print_timing: id 0 | task 0 | eval time = 17.01 ms / 2 tokens ( 17.01 ms per token, 58.79 tokens per second) +0.01.969.744 I slot print_timing: id 0 | task 0 | total time = 118.19 ms / 30 tokens +0.01.969.745 I slot print_timing: id 0 | task 0 | graphs reused = 1 +0.01.969.767 I slot release: id 0 | task 0 | stop processing: n_tokens = 29, truncated = 0 +0.01.969.774 I srv update_slots: all slots are idle +0.02.009.183 I srv server_strea: conv_id= (empty=1) +0.02.010.597 I srv operator(): chat format: peg-native +0.02.010.687 I slot get_availabl: id 0 | task -1 | - checking sim = 0.005 (12/2463) > 0.100 +0.02.010.691 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = 1551080 +0.02.010.837 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.02.010.847 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.02.010.849 I slot launch_slot_: id 0 | task 5 | processing task, is_child = 0 +0.02.010.854 I slot operator(): id 0 | task 5 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.02.010.857 I slot operator(): id 0 | task 5 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.02.010.858 I slot operator(): id 0 | task 5 | erased invalidated context checkpoint (pos_min = 0, pos_max = 23, n_tokens = 24, n_swa = 512, pos_next = 0, size = 0.985 MiB) +0.02.010.860 I slot operator(): id 0 | task 5 | cached n_tokens = 0, memory_seq_rm [0, end) +0.02.013.854 I slot operator(): id 0 | task 5 | cached n_tokens = 10, memory_seq_rm [10, end) +0.02.047.685 I slot create_check: id 0 | task 5 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.03.069.838 I slot operator(): id 0 | task 5 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.03.481.171 I slot create_check: id 0 | task 5 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.03.488.685 I slot operator(): id 0 | task 5 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.03.489.090 I slot init_sampler: id 0 | task 5 | init sampler, took 0.36 ms, tokens: text = 2463, total = 2463 +0.03.962.599 I slot create_check: id 0 | task 5 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.04.064.552 I slot print_timing: id 0 | task 5 | prompt eval time = 1978.92 ms / 2463 tokens ( 0.80 ms per token, 1244.62 tokens per second) +0.04.064.565 I slot print_timing: id 0 | task 5 | eval time = 74.75 ms / 5 tokens ( 18.69 ms per token, 53.51 tokens per second) +0.04.064.567 I slot print_timing: id 0 | task 5 | total time = 2053.67 ms / 2468 tokens +0.04.064.569 I slot print_timing: id 0 | task 5 | graphs reused = 4 +0.04.064.655 I slot release: id 0 | task 5 | stop processing: n_tokens = 2467, truncated = 0 +0.04.064.675 I srv update_slots: all slots are idle +0.04.116.072 I srv server_strea: conv_id= (empty=1) +0.04.117.531 I srv operator(): chat format: peg-native +0.04.117.673 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.04.117.692 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.04.118.058 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.04.118.096 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.04.118.103 I slot launch_slot_: id 0 | task 14 | processing task, is_child = 0 +0.04.118.126 I slot operator(): id 0 | task 14 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.04.118.134 I slot operator(): id 0 | task 14 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.04.118.139 I slot operator(): id 0 | task 14 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.04.119.166 I slot operator(): id 0 | task 14 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.04.120.012 I slot operator(): id 0 | task 14 | cached n_tokens = 0, memory_seq_rm [0, end) +0.04.124.559 I slot operator(): id 0 | task 14 | cached n_tokens = 10, memory_seq_rm [10, end) +0.04.159.959 I slot create_check: id 0 | task 14 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.05.150.735 I slot operator(): id 0 | task 14 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.05.570.992 I slot create_check: id 0 | task 14 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.05.578.158 I slot operator(): id 0 | task 14 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.05.578.603 I slot init_sampler: id 0 | task 14 | init sampler, took 0.41 ms, tokens: text = 2463, total = 2463 +0.06.056.548 I slot create_check: id 0 | task 14 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.06.155.214 I slot print_timing: id 0 | task 14 | prompt eval time = 1965.22 ms / 2463 tokens ( 0.80 ms per token, 1253.30 tokens per second) +0.06.155.223 I slot print_timing: id 0 | task 14 | eval time = 71.86 ms / 5 tokens ( 17.96 ms per token, 55.67 tokens per second) +0.06.155.225 I slot print_timing: id 0 | task 14 | total time = 2037.07 ms / 2468 tokens +0.06.155.226 I slot print_timing: id 0 | task 14 | graphs reused = 7 +0.06.155.313 I slot release: id 0 | task 14 | stop processing: n_tokens = 2467, truncated = 0 +0.06.155.336 I srv update_slots: all slots are idle +0.06.227.870 I srv server_strea: conv_id= (empty=1) +0.06.228.141 I srv operator(): chat format: peg-native +0.06.228.307 I slot get_availabl: id 0 | task -1 | - checking sim = 0.429 (12/28) > 0.100 +0.06.228.314 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 0.429 (> 0.100 thold), f_keep = 0.005 +0.06.228.522 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.06.228.539 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.06.228.542 I slot launch_slot_: id 0 | task 23 | processing task, is_child = 0 +0.06.228.550 I slot operator(): id 0 | task 23 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 28 +0.06.228.554 I slot operator(): id 0 | task 23 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.06.228.559 I slot operator(): id 0 | task 23 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.06.229.417 I slot operator(): id 0 | task 23 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.06.230.454 I slot operator(): id 0 | task 23 | cached n_tokens = 0, memory_seq_rm [0, end) +0.06.234.943 I slot operator(): id 0 | task 23 | cached n_tokens = 10, memory_seq_rm [10, end) +0.06.270.846 I slot create_check: id 0 | task 23 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.06.273.601 I slot operator(): id 0 | task 23 | cached n_tokens = 24, memory_seq_rm [24, end) +0.06.273.639 I slot init_sampler: id 0 | task 23 | init sampler, took 0.01 ms, tokens: text = 28, total = 28 +0.06.310.615 I slot create_check: id 0 | task 23 | created context checkpoint 2 of 32 (pos_min = 0, pos_max = 23, n_tokens = 24, size = 0.985 MiB) +0.06.349.481 I slot print_timing: id 0 | task 23 | prompt eval time = 102.38 ms / 28 tokens ( 3.66 ms per token, 273.50 tokens per second) +0.06.349.494 I slot print_timing: id 0 | task 23 | eval time = 18.53 ms / 2 tokens ( 18.53 ms per token, 53.96 tokens per second) +0.06.349.495 I slot print_timing: id 0 | task 23 | total time = 120.91 ms / 30 tokens +0.06.349.497 I slot print_timing: id 0 | task 23 | graphs reused = 7 +0.06.349.533 I slot release: id 0 | task 23 | stop processing: n_tokens = 29, truncated = 0 +0.06.349.560 I srv update_slots: all slots are idle diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_public_realconfig_20260923.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_public_realconfig_20260923.driver.log new file mode 100644 index 000000000..7263da52b --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_public_realconfig_20260923.driver.log @@ -0,0 +1,51 @@ +INFO 09-23 20:21:41 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 20:21:41 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 20:21:45 [omni_base.py:216] [AsyncOmni] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 +INFO 09-23 20:21:45 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 +ERROR 09-23 20:21:45 [repo_utils.py:119] Error parsing config for \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16: The repository \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 contains custom code which must be executed to correctly load the model. You can inspect the repository content at \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 . + +ERROR 09-23 20:21:45 [repo_utils.py:119] You can inspect the repository content at https://hf.co/\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16. + +ERROR 09-23 20:21:45 [repo_utils.py:119] Please pass the argument `trust_remote_code=True` to allow custom code to be run., retrying 1 of 2 +ERROR 09-23 20:21:47 [repo_utils.py:117] Error parsing config for \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16: The repository \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 contains custom code which must be executed to correctly load the model. You can inspect the repository content at \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 . + +ERROR 09-23 20:21:47 [repo_utils.py:117] You can inspect the repository content at https://hf.co/\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16. + +ERROR 09-23 20:21:47 [repo_utils.py:117] Please pass the argument `trust_remote_code=True` to allow custom code to be run. +INFO 09-23 20:21:47 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages +INFO 09-23 20:21:48 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +INFO 09-23 20:21:48 [initialization.py:356] Loaded OmniTransferConfig with 0 connector configurations +INFO 09-23 20:21:51 [orchestrator.py:604] [Orchestrator] Starting event loop +INFO 09-23 20:21:51 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages +INFO 09-23 20:21:51 [omni_base.py:235] [AsyncOmni] AsyncOmniEngine initialized in 5.90 seconds +INFO 09-23 20:21:51 [omni_base.py:255] [AsyncOmni] Initialized with 1 stages for model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\Spark-X2.5-1.7B-BF16 +INFO 09-23 20:21:51 [omni_base.py:781] [AsyncOmni] Shutting down +INFO 09-23 20:21:51 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator +INFO 09-23 20:21:51 [orchestrator.py:705] [Orchestrator] Received shutdown signal +INFO 09-23 20:21:51 [orchestrator.py:3092] [Orchestrator] Shutting down all 1 client(s) +INFO 09-23 20:21:51 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down +{ + "device": "Vulkan1", + "entrypoint": "AsyncOmni.generate", + "startup_s": 5.907832600059919, + "request_wall_s": 0.13778969994746149, + "outputs": [ + { + "request_id": "public-spark-1", + "text": "Paris", + "error": null, + "stage_event": { + "request_id": "public-spark-1-a676a073", + "stage_id": 0, + "epoch": 1, + "seq": 1, + "kind": "text", + "worker_generation": "851b8a6a700945dfbed8debbc14363e1", + "buffers": [], + "terminal": true, + "error": null + } + } + ], + "status": "passed" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_public_realconfig_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_public_realconfig_20260923.json new file mode 100644 index 000000000..8fb1ba451 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_public_realconfig_20260923.json @@ -0,0 +1,25 @@ +{ + "device": "Vulkan1", + "entrypoint": "AsyncOmni.generate", + "startup_s": 5.907832600059919, + "request_wall_s": 0.13778969994746149, + "outputs": [ + { + "request_id": "public-spark-1", + "text": "Paris", + "error": null, + "stage_event": { + "request_id": "public-spark-1-a676a073", + "stage_id": 0, + "epoch": 1, + "seq": 1, + "kind": "text", + "worker_generation": "851b8a6a700945dfbed8debbc14363e1", + "buffers": [], + "terminal": true, + "error": null + } + } + ], + "status": "passed" +} \ No newline at end of file diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_public_realconfig_20260923.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_public_realconfig_20260923.log new file mode 100644 index 000000000..0f3afd42f --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_public_realconfig_20260923.log @@ -0,0 +1,215 @@ +0.00.000.703 I srv llama_server: initializing ... +0.00.662.889 I cmn common_param: common_params_print_info: build 11124 (1a679828f) with Clang 20.1.8 for Windows x86_64 +0.00.662.895 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg) +0.00.662.896 I cmn common_param: device_info: +0.00.666.260 I cmn common_param: - Vulkan0 : NVIDIA GeForce RTX 5090 Laptop GPU (24137 MiB, 23369 MiB free) +0.00.669.265 I cmn common_param: - Vulkan1 : AMD Radeon(TM) 890M Graphics (48505 MiB, 46080 MiB free) +0.00.669.271 I cmn common_param: - CPU : AMD Ryzen AI 9 HX 370 w/ Radeon 890M (64633 MiB, 30660 MiB free) +0.00.669.323 I cmn common_param: system_info: n_threads = 12 (n_threads_batch = 12) / 24 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.669.395 I srv init_listene: running without SSL +0.00.669.452 I srv init_listene: The UI is disabled +0.00.669.452 I srv init_listene: Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable +0.00.669.455 I srv init: using 23 threads for HTTP server +0.00.669.515 W srv llama_server: security: no API key is set and CORS allows all origins (see https://github.com/ggml-org/llama.cpp/pull/25655) +0.00.675.347 I srv load_model: loading model 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' +0.00.675.354 I srv load_model: local path 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' +0.00.699.163 I llama_model_loader: loaded meta data with 37 key-value pairs and 226 tensors from C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf (version GGUF V3 (latest)) +0.00.699.175 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +0.00.699.182 I llama_model_loader: - kv 0: general.architecture str = spark2_5 +0.00.699.183 I llama_model_loader: - kv 1: general.type str = model +0.00.699.186 I llama_model_loader: - kv 2: general.sampling.top_k i32 = 0 +0.00.699.194 I llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +0.00.699.195 I llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +0.00.699.195 I llama_model_loader: - kv 5: general.name str = Hf_Format +0.00.699.196 I llama_model_loader: - kv 6: general.size_label str = 1.7B +0.00.699.199 I llama_model_loader: - kv 7: spark2_5.block_count u32 = 28 +0.00.699.199 I llama_model_loader: - kv 8: spark2_5.context_length u32 = 1048576 +0.00.699.200 I llama_model_loader: - kv 9: spark2_5.embedding_length u32 = 2048 +0.00.699.200 I llama_model_loader: - kv 10: spark2_5.feed_forward_length u32 = 6656 +0.00.699.200 I llama_model_loader: - kv 11: spark2_5.attention.head_count u32 = 8 +0.00.699.201 I llama_model_loader: - kv 12: spark2_5.attention.head_count_kv u32 = 2 +0.00.699.202 I llama_model_loader: - kv 13: spark2_5.rope.freq_base f32 = 5000000.000000 +0.00.699.203 I llama_model_loader: - kv 14: spark2_5.rope.freq_base_swa f32 = 10000.000000 +0.00.699.204 I llama_model_loader: - kv 15: spark2_5.attention.layer_norm_rms_epsilon f32 = 0.000001 +0.00.699.205 I llama_model_loader: - kv 16: spark2_5.attention.key_length u32 = 256 +0.00.699.205 I llama_model_loader: - kv 17: spark2_5.attention.value_length u32 = 256 +0.00.699.206 I llama_model_loader: - kv 18: spark2_5.vocab_size u32 = 131072 +0.00.699.206 I llama_model_loader: - kv 19: spark2_5.attention.sliding_window u32 = 512 +0.00.699.227 I llama_model_loader: - kv 20: spark2_5.attention.sliding_window_pattern arr[bool,28] = [true, true, true, false, true, true,... +0.00.699.227 I llama_model_loader: - kv 21: spark2_5.rope.dimension_count u32 = 64 +0.00.699.228 I llama_model_loader: - kv 22: spark2_5.rope.dimension_count_swa u32 = 256 +0.00.699.228 I llama_model_loader: - kv 23: tokenizer.ggml.model str = gpt2 +0.00.699.229 I llama_model_loader: - kv 24: tokenizer.ggml.pre str = spark2_5 +0.00.714.722 I llama_model_loader: - kv 25: tokenizer.ggml.tokens arr[str,131072] = ["<｜start▁of▁sentence｜>", "<... +0.00.719.773 I llama_model_loader: - kv 26: tokenizer.ggml.token_type arr[i32,131072] = [3, 3, 3, 4, 4, 3, 3, 3, 4, 4, 4, 3, ... +0.00.735.489 I llama_model_loader: - kv 27: tokenizer.ggml.merges arr[str,130716] = ["Ġ Ġ", "Ġ t", "i n", "Ġ a", "Ġ... +0.00.735.497 I llama_model_loader: - kv 28: tokenizer.ggml.bos_token_id u32 = 0 +0.00.735.497 I llama_model_loader: - kv 29: tokenizer.ggml.eos_token_id u32 = 1 +0.00.735.498 I llama_model_loader: - kv 30: tokenizer.ggml.unknown_token_id u32 = 5 +0.00.735.498 I llama_model_loader: - kv 31: tokenizer.ggml.padding_token_id u32 = 2 +0.00.735.499 I llama_model_loader: - kv 32: tokenizer.ggml.add_bos_token bool = false +0.00.735.499 I llama_model_loader: - kv 33: tokenizer.ggml.add_eos_token bool = false +0.00.735.505 I llama_model_loader: - kv 34: tokenizer.chat_template str = {%- if not messages %}\n {{- raise_... +0.00.735.506 I llama_model_loader: - kv 35: general.quantization_version u32 = 2 +0.00.735.506 I llama_model_loader: - kv 36: general.file_type u32 = 15 +0.00.735.515 I llama_model_loader: - type f32: 57 tensors +0.00.735.515 I llama_model_loader: - type q4_K: 140 tensors +0.00.735.516 I llama_model_loader: - type q6_K: 29 tensors +0.00.735.520 I print_info: file format = GGUF V3 (latest) +0.00.735.523 I print_info: file type = Q4_K - Medium +0.00.735.528 I print_info: file size = 1.03 GiB (5.16 BPW) +0.00.739.225 I llama_prepare_model_devices: using device Vulkan1 (AMD Radeon(TM) 890M Graphics) (unknown id) - 46080 MiB free +0.00.783.572 I load: 86 unused tokens +0.00.798.261 I load: printing all EOG tokens: +0.00.798.266 I load: - 1 ('<｜end▁of▁sentence｜>') +0.00.798.374 I load: special tokens cache size = 113 +0.00.813.738 I load: token to piece cache size = 0.7975 MB +0.00.813.762 I print_info: arch = spark2_5 +0.00.813.762 I print_info: vocab_only = 0 +0.00.813.762 I print_info: no_alloc = 0 +0.00.813.763 I print_info: n_ctx_train = 1048576 +0.00.813.764 I print_info: n_embd_inp = 2048 +0.00.813.764 I print_info: n_embd = 2048 +0.00.813.764 I print_info: n_embd_out = 2048 +0.00.813.765 I print_info: n_layer = 28 +0.00.813.765 I print_info: n_layer_all = 28 +0.00.813.782 I print_info: n_head = 8 +0.00.813.783 I print_info: n_head_kv = 2 +0.00.813.784 I print_info: n_rot = 64 +0.00.813.784 I print_info: n_swa = 512 +0.00.813.784 I print_info: is_swa_any = 1 +0.00.813.785 I print_info: non_causal_type = 0 +0.00.813.785 I print_info: n_embd_head_k = 256 +0.00.813.785 I print_info: n_embd_head_v = 256 +0.00.813.786 I print_info: n_gqa = 4 +0.00.813.788 I print_info: n_embd_k_gqa = 512 +0.00.813.789 I print_info: n_embd_v_gqa = 512 +0.00.813.791 I print_info: f_norm_eps = 0.0e+00 +0.00.813.792 I print_info: f_norm_rms_eps = 1.0e-06 +0.00.813.792 I print_info: f_clamp_kqv = 0.0e+00 +0.00.813.793 I print_info: f_max_alibi_bias = 0.0e+00 +0.00.813.793 I print_info: f_logit_scale = 0.0e+00 +0.00.813.793 I print_info: f_attn_scale = 0.0e+00 +0.00.813.794 I print_info: f_attn_value_scale = 0.0000 +0.00.813.795 I print_info: n_ff = 6656 +0.00.813.796 I print_info: n_expert = 0 +0.00.813.796 I print_info: n_expert_used = 0 +0.00.813.798 I print_info: n_expert_groups = 0 +0.00.813.798 I print_info: n_group_used = 0 +0.00.813.798 I print_info: causal attn = 1 +0.00.813.799 I print_info: pooling type = -1 +0.00.813.799 I print_info: rope type = 2 +0.00.813.799 I print_info: rope scaling = linear +0.00.813.800 I print_info: freq_base_train = 5000000.0 +0.00.813.802 I print_info: freq_scale_train = 1 +0.00.813.803 I print_info: freq_base_swa = 10000.0 +0.00.813.803 I print_info: freq_scale_swa = 1 +0.00.813.803 I print_info: n_embd_head_k_swa = 256 +0.00.813.804 I print_info: n_embd_head_v_swa = 256 +0.00.813.804 I print_info: n_rot_swa = 256 +0.00.813.804 I print_info: n_ctx_orig_yarn = 1048576 +0.00.813.805 I print_info: rope_yarn_log_mul = 0.0000 +0.00.813.805 I print_info: rope_finetuned = unknown +0.00.813.806 I print_info: model type = 1.7B +0.00.813.807 I print_info: model params = 1.71 B +0.00.813.807 I print_info: general.name = Hf_Format +0.00.813.808 I print_info: vocab type = BPE +0.00.813.808 I print_info: n_vocab = 131072 +0.00.813.808 I print_info: n_merges = 130716 +0.00.813.809 I print_info: BOS token = 0 '<｜start▁of▁sentence｜>' +0.00.813.810 I print_info: EOS token = 1 '<｜end▁of▁sentence｜>' +0.00.813.810 I print_info: EOT token = 1 '<｜end▁of▁sentence｜>' +0.00.813.811 I print_info: UNK token = 5 '' +0.00.813.811 I print_info: PAD token = 2 '<｜▁pad▁｜>' +0.00.813.811 I print_info: LF token = 198 'Ċ' +0.00.813.812 I print_info: FIM PRE token = 11 '<｜fim▁begin｜>' +0.00.813.812 I print_info: FIM SUF token = 12 '<｜fim▁hole｜>' +0.00.813.813 I print_info: FIM MID token = 13 '<｜fim▁end｜>' +0.00.813.813 I print_info: EOG token = 1 '<｜end▁of▁sentence｜>' +0.00.813.814 I print_info: max token length = 210 +0.00.821.645 I load_tensors: loading model tensors, this can take a while... (load_mode = none) +0.01.423.544 I load_tensors: offloading output layer to GPU +0.01.423.549 I load_tensors: offloading 27 repeating layers to GPU +0.01.423.550 I load_tensors: offloaded 29/29 layers to GPU +0.01.423.555 I load_tensors: Vulkan1 model buffer size = 1051.02 MiB +0.01.423.556 I load_tensors: Vulkan_Host model buffer size = 210.00 MiB +0.01.784.106 I cmn common_init_: added <｜end▁of▁sentence｜> logit bias = -inf +0.01.784.476 I llama_context: constructing llama_context +0.01.784.483 I llama_context: n_seq_max = 1 +0.01.784.484 I llama_context: n_ctx = 4096 +0.01.784.484 I llama_context: n_ctx_seq = 4096 +0.01.784.484 I llama_context: n_batch = 2048 +0.01.784.485 I llama_context: n_ubatch = 512 +0.01.784.485 I llama_context: causal_attn = 1 +0.01.784.485 I llama_context: flash_attn = auto +0.01.784.486 I llama_context: kv_unified = false +0.01.784.489 I llama_context: freq_base = 5000000.0 +0.01.784.491 I llama_context: freq_scale = 1 +0.01.784.492 I llama_context: n_rs_seq = 0 +0.01.784.492 I llama_context: n_outputs_max = 1 +0.01.784.492 I llama_context: n_outputs_max_per_seq = 1 +0.01.784.493 I llama_context: n_ctx_seq (4096) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized +0.01.784.814 I llama_context: Vulkan_Host output buffer size = 0.50 MiB +0.01.784.821 I llama_kv_cache_iswa: creating non-SWA KV cache, size = 4096 cells +0.01.789.518 I llama_kv_cache: Vulkan1 KV buffer size = 56.00 MiB +0.01.790.047 I llama_kv_cache: size = 56.00 MiB ( 4096 cells, 7 layers, 1/1 seqs), K (f16): 28.00 MiB, V (f16): 28.00 MiB +0.01.790.053 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.01.790.053 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.01.790.057 I llama_kv_cache_iswa: creating SWA KV cache, size = 1024 cells +0.01.793.423 I llama_kv_cache: Vulkan1 KV buffer size = 42.00 MiB +0.01.793.815 I llama_kv_cache: size = 42.00 MiB ( 1024 cells, 21 layers, 1/1 seqs), K (f16): 21.00 MiB, V (f16): 21.00 MiB +0.01.793.820 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.01.793.820 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.01.793.826 I sched_reserve: reserving ... +0.01.795.336 I resolve_fused_ops: Flash Attention enabled +0.01.795.341 I resolve_fused_ops: resolving fused DeepSeek V4 HC support: +0.01.796.216 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled +0.01.797.054 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled +0.01.797.869 I resolve_fused_ops: fused DeepSeek V4 HC post enabled +0.01.805.423 I sched_reserve: Vulkan1 compute buffer size = 56.02 MiB +0.01.805.427 I sched_reserve: Vulkan_Host compute buffer size = 13.02 MiB +0.01.805.429 I sched_reserve: graph: nodes = 986, splits = 2, input objects = 4, input tensors = 9 +0.01.805.431 I sched_reserve: reserve took 11.60 ms, sched copies = 1 +0.01.805.520 I cmn init: llama threadpool init, n_threads = 12 +0.01.805.534 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.01.852.133 I srv load_model: initializing, n_slots = 1, n_ctx_slot = 4096, kv_unified = 'false' +0.01.852.148 I spec common_specu: no implementations specified for speculative decoding +0.01.852.149 I slot load_model: id 0 | task -1 | new slot, n_ctx = 4096 +0.01.852.222 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it +0.01.852.224 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.01.852.225 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192 +0.01.852.244 W srv init: --cache-idle-slots requires --cache-ram, disabling +0.01.861.705 I srv init: init: chat template, example_format: '<｜start▁of▁sentence｜><|System|> +you are a helpful assistant. + +You are a helpful assistant<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>Hello<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>Hi there<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>How are you?<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>' +0.01.868.556 I srv init: init: chat template, thinking = 0 +0.01.868.563 I srv init: preserve_reasoning kwarg: true +0.01.868.564 W srv init: chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve) +0.01.868.580 I srv llama_server: model loaded +0.01.868.583 I srv llama_server: listening on http://127.0.0.1:65098 +0.01.868.655 I srv update_slots: all slots are idle +0.02.065.256 I srv server_strea: conv_id= (empty=1) +0.02.065.443 I srv operator(): chat format: peg-native +0.02.065.582 I slot get_availabl: id 0 | task -1 | - skipping, slot is empty +0.02.065.593 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.02.065.665 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.02.065.698 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.02.065.706 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.02.065.723 I slot operator(): id 0 | task 0 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 28 +0.02.065.734 I slot operator(): id 0 | task 0 | cached n_tokens = 0, memory_seq_rm [0, end) +0.02.075.772 I slot operator(): id 0 | task 0 | cached n_tokens = 10, memory_seq_rm [10, end) +0.02.107.568 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.02.110.384 I slot operator(): id 0 | task 0 | cached n_tokens = 24, memory_seq_rm [24, end) +0.02.110.408 I slot init_sampler: id 0 | task 0 | init sampler, took 0.00 ms, tokens: text = 28, total = 28 +0.02.146.523 I slot create_check: id 0 | task 0 | created context checkpoint 2 of 32 (pos_min = 0, pos_max = 23, n_tokens = 24, size = 0.985 MiB) +0.02.182.869 I slot print_timing: id 0 | task 0 | prompt eval time = 100.89 ms / 28 tokens ( 3.60 ms per token, 277.53 tokens per second) +0.02.182.874 I slot print_timing: id 0 | task 0 | eval time = 16.24 ms / 2 tokens ( 16.24 ms per token, 61.56 tokens per second) +0.02.182.875 I slot print_timing: id 0 | task 0 | total time = 117.13 ms / 30 tokens +0.02.182.875 I slot print_timing: id 0 | task 0 | graphs reused = 1 +0.02.182.897 I slot release: id 0 | task 0 | stop processing: n_tokens = 29, truncated = 0 +0.02.182.902 I srv update_slots: all slots are idle diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_tokenbound_20260923.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_tokenbound_20260923.driver.log new file mode 100644 index 000000000..6c6c0ea9a --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_tokenbound_20260923.driver.log @@ -0,0 +1,171 @@ +INFO 09-23 20:19:58 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 20:19:58 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 20:20:01 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +WARNING 09-23 20:20:01 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 13] Permission denied: 'C:\\Users\\zhout\\w2' +{ + "scope": "complete Spark GGUF text requests through Omni StageRuntime/StagePool and external llama.cpp backend", + "model_file": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "device": "Vulkan1", + "memory_budget": { + "capacities": { + "host_ram": 12884901888 + }, + "demands": { + "host_ram": 4294967296 + } + }, + "warmup_count": 1, + "measured_count": 20, + "startup_s": 3.119025900028646, + "execution_plan": { + "backend": "external.llamacpp.text.v1", + "stage_id": 0, + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "model_sha256": "902bde2522394954ac17821b3e5fd0df02defbc6944f122253f2580acf0503f4", + "server_sha256": "9ffc5919acb4cb43c7be5f3053b59014cce70a87d3a801fcad49c4f459984f52", + "model_alias": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "requested_device": "Vulkan1", + "expected_device_name": "AMD Radeon(TM) 890M Graphics", + "layer_assignments": 0, + "offloaded_layers": [ + "29", + "29" + ], + "model_buffer_devices": [ + "Vulkan1" + ], + "worker_pid": 53444, + "loaded_rss_bytes": 1640730624, + "reserved_bytes": { + "host_ram": 4294967296 + }, + "memory_pool": "host_ram", + "memory_overhead_bytes": 2147483648, + "context_tokens": 4096, + "max_new_tokens": 96, + "max_io_bytes": 1048576, + "request_capacity": 1, + "stateful_session": "llama.cpp-owned; reset for each complete request", + "evidence": "B" + }, + "checks": [ + { + "case": "short", + "answer": "Paris", + "expected": "Paris", + "correct": true, + "wall_s": 0.1641275999136269, + "stage_event": { + "request_id": "short-620738921000000", + "stage_id": 0, + "epoch": 1, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 0.14191909995861351, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 28, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.1124441999709234, + "stage_event": { + "request_id": "inventory_120-620739093000000", + "stage_id": 0, + "epoch": 2, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.108132400084287, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "nearest_rank_p50_wall_s": 2.1021421999903396, + "nearest_rank_p95_wall_s": 2.1353829000145197, + "context_check": { + "rejected": true, + "error": "ResourceUnavailable: Spark prompt plus generation bound exceeds context; refusing truncation", + "recovery": { + "case": "short", + "answer": "Paris", + "expected": "Paris", + "correct": true, + "wall_s": 0.15485639998223633, + "stage_event": { + "request_id": "short-620785484000000", + "stage_id": 0, + "epoch": 24, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 0.14172140008304268, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 28, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + }, + "abort_check": { + "stale_output": false, + "worker_exited": true, + "ledger_after_abort": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } + }, + "status": "passed", + "ledger_after_shutdown": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_tokenbound_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_tokenbound_20260923.json new file mode 100644 index 000000000..642f7ec96 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_tokenbound_20260923.json @@ -0,0 +1,780 @@ +{ + "scope": "complete Spark GGUF text requests through Omni StageRuntime/StagePool and external llama.cpp backend", + "model_file": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "device": "Vulkan1", + "memory_budget": { + "capacities": { + "host_ram": 12884901888 + }, + "demands": { + "host_ram": 4294967296 + } + }, + "warmup_count": 1, + "measured_count": 20, + "startup_s": 3.119025900028646, + "execution_plan": { + "backend": "external.llamacpp.text.v1", + "stage_id": 0, + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "model_sha256": "902bde2522394954ac17821b3e5fd0df02defbc6944f122253f2580acf0503f4", + "server_sha256": "9ffc5919acb4cb43c7be5f3053b59014cce70a87d3a801fcad49c4f459984f52", + "model_alias": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "requested_device": "Vulkan1", + "expected_device_name": "AMD Radeon(TM) 890M Graphics", + "layer_assignments": 0, + "offloaded_layers": [ + "29", + "29" + ], + "model_buffer_devices": [ + "Vulkan1" + ], + "worker_pid": 53444, + "loaded_rss_bytes": 1640730624, + "reserved_bytes": { + "host_ram": 4294967296 + }, + "memory_pool": "host_ram", + "memory_overhead_bytes": 2147483648, + "context_tokens": 4096, + "max_new_tokens": 96, + "max_io_bytes": 1048576, + "request_capacity": 1, + "stateful_session": "llama.cpp-owned; reset for each complete request", + "evidence": "B" + }, + "checks": [ + { + "case": "short", + "answer": "Paris", + "expected": "Paris", + "correct": true, + "wall_s": 0.1641275999136269, + "stage_event": { + "request_id": "short-620738921000000", + "stage_id": 0, + "epoch": 1, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 0.14191909995861351, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 28, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.1124441999709234, + "stage_event": { + "request_id": "inventory_120-620739093000000", + "stage_id": 0, + "epoch": 2, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.108132400084287, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "warmups": [ + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.1003702000016347, + "stage_event": { + "request_id": "inventory_120-620741203000000", + "stage_id": 0, + "epoch": 3, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.079685500008054, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "measured": [ + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.0663093000184745, + "stage_event": { + "request_id": "inventory_120-620743312000000", + "stage_id": 0, + "epoch": 4, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.060538399964571, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.0768924999283627, + "stage_event": { + "request_id": "inventory_120-620745375000000", + "stage_id": 0, + "epoch": 5, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.07273230003193, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.116518099908717, + "stage_event": { + "request_id": "inventory_120-620747453000000", + "stage_id": 0, + "epoch": 6, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.0932353999232873, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.1021421999903396, + "stage_event": { + "request_id": "inventory_120-620749562000000", + "stage_id": 0, + "epoch": 7, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.070734800072387, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.083520899992436, + "stage_event": { + "request_id": "inventory_120-620751671000000", + "stage_id": 0, + "epoch": 8, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.0777252999832854, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.0945620000129566, + "stage_event": { + "request_id": "inventory_120-620753750000000", + "stage_id": 0, + "epoch": 9, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.0759557000128552, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.072307400056161, + "stage_event": { + "request_id": "inventory_120-620755843000000", + "stage_id": 0, + "epoch": 10, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.0674628999549896, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.162337200017646, + "stage_event": { + "request_id": "inventory_120-620757921000000", + "stage_id": 0, + "epoch": 11, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.132587500032969, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.1250597999896854, + "stage_event": { + "request_id": "inventory_120-620760078000000", + "stage_id": 0, + "epoch": 12, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.0997529999585822, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.1086074999766424, + "stage_event": { + "request_id": "inventory_120-620762203000000", + "stage_id": 0, + "epoch": 13, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.08812719990965, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.0795605999883264, + "stage_event": { + "request_id": "inventory_120-620764312000000", + "stage_id": 0, + "epoch": 14, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.0734709999524057, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.1347006999421865, + "stage_event": { + "request_id": "inventory_120-620766390000000", + "stage_id": 0, + "epoch": 15, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.112379099940881, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.1353829000145197, + "stage_event": { + "request_id": "inventory_120-620768531000000", + "stage_id": 0, + "epoch": 16, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.110274499980733, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.0979999999981374, + "stage_event": { + "request_id": "inventory_120-620770671000000", + "stage_id": 0, + "epoch": 17, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.0931969999801368, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.1295480000553653, + "stage_event": { + "request_id": "inventory_120-620772765000000", + "stage_id": 0, + "epoch": 18, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.1082848999649286, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.085711099905893, + "stage_event": { + "request_id": "inventory_120-620774890000000", + "stage_id": 0, + "epoch": 19, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.0792751000262797, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.094759300001897, + "stage_event": { + "request_id": "inventory_120-620776984000000", + "stage_id": 0, + "epoch": 20, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.0900978000136092, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.1228663000511006, + "stage_event": { + "request_id": "inventory_120-620779078000000", + "stage_id": 0, + "epoch": 21, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.1064914000453427, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.132605299935676, + "stage_event": { + "request_id": "inventory_120-620781203000000", + "stage_id": 0, + "epoch": 22, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.1088262000121176, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + }, + { + "case": "inventory_120", + "answer": "1511", + "expected": "1511", + "correct": true, + "wall_s": 2.11771470005624, + "stage_event": { + "request_id": "inventory_120-620783328000000", + "stage_id": 0, + "epoch": 23, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 2.094285300001502, + "usage": { + "completion_tokens": 5, + "prompt_tokens": 2463, + "total_tokens": 2468, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + ], + "nearest_rank_p50_wall_s": 2.1021421999903396, + "nearest_rank_p95_wall_s": 2.1353829000145197, + "context_check": { + "rejected": true, + "error": "ResourceUnavailable: Spark prompt plus generation bound exceeds context; refusing truncation", + "recovery": { + "case": "short", + "answer": "Paris", + "expected": "Paris", + "correct": true, + "wall_s": 0.15485639998223633, + "stage_event": { + "request_id": "short-620785484000000", + "stage_id": 0, + "epoch": 24, + "seq": 1, + "kind": "text", + "worker_generation": "ed06285050014c51be39c947422f9a4f", + "buffers": [], + "terminal": true, + "error": null + }, + "metrics": { + "llamacpp_wall_s": 0.14172140008304268, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 28, + "total_tokens": 30, + "prompt_tokens_details": { + "cached_tokens": 0 + } + } + } + } + }, + "abort_check": { + "stale_output": false, + "worker_exited": true, + "ledger_after_abort": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } + }, + "status": "passed", + "ledger_after_shutdown": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_tokenbound_20260923.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_tokenbound_20260923.log new file mode 100644 index 000000000..d09cf9f7d --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_radeon_tokenbound_20260923.log @@ -0,0 +1,879 @@ +0.00.000.725 I srv llama_server: initializing ... +0.00.163.614 I cmn common_param: common_params_print_info: build 11124 (1a679828f) with Clang 20.1.8 for Windows x86_64 +0.00.163.620 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg) +0.00.163.621 I cmn common_param: device_info: +0.00.167.234 I cmn common_param: - Vulkan0 : NVIDIA GeForce RTX 5090 Laptop GPU (24137 MiB, 23369 MiB free) +0.00.170.350 I cmn common_param: - Vulkan1 : AMD Radeon(TM) 890M Graphics (48505 MiB, 46080 MiB free) +0.00.170.358 I cmn common_param: - CPU : AMD Ryzen AI 9 HX 370 w/ Radeon 890M (64633 MiB, 30624 MiB free) +0.00.170.418 I cmn common_param: system_info: n_threads = 12 (n_threads_batch = 12) / 24 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.170.497 I srv init_listene: running without SSL +0.00.170.563 I srv init_listene: The UI is disabled +0.00.170.564 I srv init_listene: Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable +0.00.170.568 I srv init: using 23 threads for HTTP server +0.00.170.636 W srv llama_server: security: no API key is set and CORS allows all origins (see https://github.com/ggml-org/llama.cpp/pull/25655) +0.00.180.972 I srv load_model: loading model 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' +0.00.180.983 I srv load_model: local path 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' +0.00.206.237 I llama_model_loader: loaded meta data with 37 key-value pairs and 226 tensors from C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf (version GGUF V3 (latest)) +0.00.206.250 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +0.00.206.257 I llama_model_loader: - kv 0: general.architecture str = spark2_5 +0.00.206.258 I llama_model_loader: - kv 1: general.type str = model +0.00.206.262 I llama_model_loader: - kv 2: general.sampling.top_k i32 = 0 +0.00.206.269 I llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +0.00.206.270 I llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +0.00.206.271 I llama_model_loader: - kv 5: general.name str = Hf_Format +0.00.206.271 I llama_model_loader: - kv 6: general.size_label str = 1.7B +0.00.206.274 I llama_model_loader: - kv 7: spark2_5.block_count u32 = 28 +0.00.206.274 I llama_model_loader: - kv 8: spark2_5.context_length u32 = 1048576 +0.00.206.275 I llama_model_loader: - kv 9: spark2_5.embedding_length u32 = 2048 +0.00.206.275 I llama_model_loader: - kv 10: spark2_5.feed_forward_length u32 = 6656 +0.00.206.276 I llama_model_loader: - kv 11: spark2_5.attention.head_count u32 = 8 +0.00.206.276 I llama_model_loader: - kv 12: spark2_5.attention.head_count_kv u32 = 2 +0.00.206.277 I llama_model_loader: - kv 13: spark2_5.rope.freq_base f32 = 5000000.000000 +0.00.206.278 I llama_model_loader: - kv 14: spark2_5.rope.freq_base_swa f32 = 10000.000000 +0.00.206.280 I llama_model_loader: - kv 15: spark2_5.attention.layer_norm_rms_epsilon f32 = 0.000001 +0.00.206.280 I llama_model_loader: - kv 16: spark2_5.attention.key_length u32 = 256 +0.00.206.281 I llama_model_loader: - kv 17: spark2_5.attention.value_length u32 = 256 +0.00.206.281 I llama_model_loader: - kv 18: spark2_5.vocab_size u32 = 131072 +0.00.206.282 I llama_model_loader: - kv 19: spark2_5.attention.sliding_window u32 = 512 +0.00.206.303 I llama_model_loader: - kv 20: spark2_5.attention.sliding_window_pattern arr[bool,28] = [true, true, true, false, true, true,... +0.00.206.304 I llama_model_loader: - kv 21: spark2_5.rope.dimension_count u32 = 64 +0.00.206.304 I llama_model_loader: - kv 22: spark2_5.rope.dimension_count_swa u32 = 256 +0.00.206.305 I llama_model_loader: - kv 23: tokenizer.ggml.model str = gpt2 +0.00.206.305 I llama_model_loader: - kv 24: tokenizer.ggml.pre str = spark2_5 +0.00.223.649 I llama_model_loader: - kv 25: tokenizer.ggml.tokens arr[str,131072] = ["<｜start▁of▁sentence｜>", "<... +0.00.228.825 I llama_model_loader: - kv 26: tokenizer.ggml.token_type arr[i32,131072] = [3, 3, 3, 4, 4, 3, 3, 3, 4, 4, 4, 3, ... +0.00.246.322 I llama_model_loader: - kv 27: tokenizer.ggml.merges arr[str,130716] = ["Ġ Ġ", "Ġ t", "i n", "Ġ a", "Ġ... +0.00.246.332 I llama_model_loader: - kv 28: tokenizer.ggml.bos_token_id u32 = 0 +0.00.246.333 I llama_model_loader: - kv 29: tokenizer.ggml.eos_token_id u32 = 1 +0.00.246.334 I llama_model_loader: - kv 30: tokenizer.ggml.unknown_token_id u32 = 5 +0.00.246.334 I llama_model_loader: - kv 31: tokenizer.ggml.padding_token_id u32 = 2 +0.00.246.337 I llama_model_loader: - kv 32: tokenizer.ggml.add_bos_token bool = false +0.00.246.337 I llama_model_loader: - kv 33: tokenizer.ggml.add_eos_token bool = false +0.00.246.343 I llama_model_loader: - kv 34: tokenizer.chat_template str = {%- if not messages %}\n {{- raise_... +0.00.246.344 I llama_model_loader: - kv 35: general.quantization_version u32 = 2 +0.00.246.345 I llama_model_loader: - kv 36: general.file_type u32 = 15 +0.00.246.352 I llama_model_loader: - type f32: 57 tensors +0.00.246.353 I llama_model_loader: - type q4_K: 140 tensors +0.00.246.354 I llama_model_loader: - type q6_K: 29 tensors +0.00.246.359 I print_info: file format = GGUF V3 (latest) +0.00.246.359 I print_info: file type = Q4_K - Medium +0.00.246.364 I print_info: file size = 1.03 GiB (5.16 BPW) +0.00.249.995 I llama_prepare_model_devices: using device Vulkan1 (AMD Radeon(TM) 890M Graphics) (unknown id) - 46080 MiB free +0.00.313.982 I load: 86 unused tokens +0.00.335.818 I load: printing all EOG tokens: +0.00.335.827 I load: - 1 ('<｜end▁of▁sentence｜>') +0.00.335.953 I load: special tokens cache size = 113 +0.00.351.163 I load: token to piece cache size = 0.7975 MB +0.00.351.184 I print_info: arch = spark2_5 +0.00.351.185 I print_info: vocab_only = 0 +0.00.351.185 I print_info: no_alloc = 0 +0.00.351.186 I print_info: n_ctx_train = 1048576 +0.00.351.186 I print_info: n_embd_inp = 2048 +0.00.351.187 I print_info: n_embd = 2048 +0.00.351.187 I print_info: n_embd_out = 2048 +0.00.351.187 I print_info: n_layer = 28 +0.00.351.187 I print_info: n_layer_all = 28 +0.00.351.206 I print_info: n_head = 8 +0.00.351.207 I print_info: n_head_kv = 2 +0.00.351.208 I print_info: n_rot = 64 +0.00.351.208 I print_info: n_swa = 512 +0.00.351.208 I print_info: is_swa_any = 1 +0.00.351.209 I print_info: non_causal_type = 0 +0.00.351.209 I print_info: n_embd_head_k = 256 +0.00.351.209 I print_info: n_embd_head_v = 256 +0.00.351.211 I print_info: n_gqa = 4 +0.00.351.212 I print_info: n_embd_k_gqa = 512 +0.00.351.213 I print_info: n_embd_v_gqa = 512 +0.00.351.215 I print_info: f_norm_eps = 0.0e+00 +0.00.351.216 I print_info: f_norm_rms_eps = 1.0e-06 +0.00.351.217 I print_info: f_clamp_kqv = 0.0e+00 +0.00.351.217 I print_info: f_max_alibi_bias = 0.0e+00 +0.00.351.217 I print_info: f_logit_scale = 0.0e+00 +0.00.351.218 I print_info: f_attn_scale = 0.0e+00 +0.00.351.218 I print_info: f_attn_value_scale = 0.0000 +0.00.351.220 I print_info: n_ff = 6656 +0.00.351.220 I print_info: n_expert = 0 +0.00.351.220 I print_info: n_expert_used = 0 +0.00.351.222 I print_info: n_expert_groups = 0 +0.00.351.223 I print_info: n_group_used = 0 +0.00.351.223 I print_info: causal attn = 1 +0.00.351.223 I print_info: pooling type = -1 +0.00.351.224 I print_info: rope type = 2 +0.00.351.225 I print_info: rope scaling = linear +0.00.351.226 I print_info: freq_base_train = 5000000.0 +0.00.351.227 I print_info: freq_scale_train = 1 +0.00.351.228 I print_info: freq_base_swa = 10000.0 +0.00.351.228 I print_info: freq_scale_swa = 1 +0.00.351.229 I print_info: n_embd_head_k_swa = 256 +0.00.351.229 I print_info: n_embd_head_v_swa = 256 +0.00.351.229 I print_info: n_rot_swa = 256 +0.00.351.230 I print_info: n_ctx_orig_yarn = 1048576 +0.00.351.230 I print_info: rope_yarn_log_mul = 0.0000 +0.00.351.230 I print_info: rope_finetuned = unknown +0.00.351.231 I print_info: model type = 1.7B +0.00.351.232 I print_info: model params = 1.71 B +0.00.351.232 I print_info: general.name = Hf_Format +0.00.351.233 I print_info: vocab type = BPE +0.00.351.234 I print_info: n_vocab = 131072 +0.00.351.234 I print_info: n_merges = 130716 +0.00.351.234 I print_info: BOS token = 0 '<｜start▁of▁sentence｜>' +0.00.351.235 I print_info: EOS token = 1 '<｜end▁of▁sentence｜>' +0.00.351.235 I print_info: EOT token = 1 '<｜end▁of▁sentence｜>' +0.00.351.235 I print_info: UNK token = 5 '' +0.00.351.236 I print_info: PAD token = 2 '<｜▁pad▁｜>' +0.00.351.236 I print_info: LF token = 198 'Ċ' +0.00.351.237 I print_info: FIM PRE token = 11 '<｜fim▁begin｜>' +0.00.351.237 I print_info: FIM SUF token = 12 '<｜fim▁hole｜>' +0.00.351.238 I print_info: FIM MID token = 13 '<｜fim▁end｜>' +0.00.351.238 I print_info: EOG token = 1 '<｜end▁of▁sentence｜>' +0.00.351.239 I print_info: max token length = 210 +0.00.357.846 I load_tensors: loading model tensors, this can take a while... (load_mode = none) +0.00.971.687 I load_tensors: offloading output layer to GPU +0.00.971.693 I load_tensors: offloading 27 repeating layers to GPU +0.00.971.694 I load_tensors: offloaded 29/29 layers to GPU +0.00.971.698 I load_tensors: Vulkan1 model buffer size = 1051.02 MiB +0.00.971.699 I load_tensors: Vulkan_Host model buffer size = 210.00 MiB +0.01.336.105 I cmn common_init_: added <｜end▁of▁sentence｜> logit bias = -inf +0.01.336.477 I llama_context: constructing llama_context +0.01.336.484 I llama_context: n_seq_max = 1 +0.01.336.485 I llama_context: n_ctx = 4096 +0.01.336.485 I llama_context: n_ctx_seq = 4096 +0.01.336.486 I llama_context: n_batch = 2048 +0.01.336.486 I llama_context: n_ubatch = 512 +0.01.336.487 I llama_context: causal_attn = 1 +0.01.336.487 I llama_context: flash_attn = auto +0.01.336.488 I llama_context: kv_unified = false +0.01.336.492 I llama_context: freq_base = 5000000.0 +0.01.336.494 I llama_context: freq_scale = 1 +0.01.336.494 I llama_context: n_rs_seq = 0 +0.01.336.495 I llama_context: n_outputs_max = 1 +0.01.336.495 I llama_context: n_outputs_max_per_seq = 1 +0.01.336.496 I llama_context: n_ctx_seq (4096) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized +0.01.336.834 I llama_context: Vulkan_Host output buffer size = 0.50 MiB +0.01.336.841 I llama_kv_cache_iswa: creating non-SWA KV cache, size = 4096 cells +0.01.341.535 I llama_kv_cache: Vulkan1 KV buffer size = 56.00 MiB +0.01.342.196 I llama_kv_cache: size = 56.00 MiB ( 4096 cells, 7 layers, 1/1 seqs), K (f16): 28.00 MiB, V (f16): 28.00 MiB +0.01.342.205 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.01.342.205 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.01.342.210 I llama_kv_cache_iswa: creating SWA KV cache, size = 1024 cells +0.01.346.316 I llama_kv_cache: Vulkan1 KV buffer size = 42.00 MiB +0.01.346.760 I llama_kv_cache: size = 42.00 MiB ( 1024 cells, 21 layers, 1/1 seqs), K (f16): 21.00 MiB, V (f16): 21.00 MiB +0.01.346.765 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.01.346.765 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.01.346.772 I sched_reserve: reserving ... +0.01.348.239 I resolve_fused_ops: Flash Attention enabled +0.01.348.242 I resolve_fused_ops: resolving fused DeepSeek V4 HC support: +0.01.349.136 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled +0.01.349.963 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled +0.01.350.777 I resolve_fused_ops: fused DeepSeek V4 HC post enabled +0.01.358.915 I sched_reserve: Vulkan1 compute buffer size = 56.02 MiB +0.01.358.920 I sched_reserve: Vulkan_Host compute buffer size = 13.02 MiB +0.01.358.922 I sched_reserve: graph: nodes = 986, splits = 2, input objects = 4, input tensors = 9 +0.01.358.923 I sched_reserve: reserve took 12.15 ms, sched copies = 1 +0.01.359.019 I cmn init: llama threadpool init, n_threads = 12 +0.01.359.035 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.01.406.764 I srv load_model: initializing, n_slots = 1, n_ctx_slot = 4096, kv_unified = 'false' +0.01.406.783 I spec common_specu: no implementations specified for speculative decoding +0.01.406.784 I slot load_model: id 0 | task -1 | new slot, n_ctx = 4096 +0.01.406.865 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it +0.01.406.867 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.01.406.868 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192 +0.01.406.889 W srv init: --cache-idle-slots requires --cache-ram, disabling +0.01.417.023 I srv init: init: chat template, example_format: '<｜start▁of▁sentence｜><|System|> +you are a helpful assistant. + +You are a helpful assistant<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>Hello<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>Hi there<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>How are you?<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>' +0.01.425.884 I srv init: init: chat template, thinking = 0 +0.01.425.898 I srv init: preserve_reasoning kwarg: true +0.01.425.899 W srv init: chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve) +0.01.425.925 I srv llama_server: model loaded +0.01.425.928 I srv llama_server: listening on http://127.0.0.1:58101 +0.01.426.010 I srv update_slots: all slots are idle +0.01.626.586 I srv server_strea: conv_id= (empty=1) +0.01.626.766 I srv operator(): chat format: peg-native +0.01.626.884 I slot get_availabl: id 0 | task -1 | - skipping, slot is empty +0.01.626.897 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.01.626.962 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.01.626.996 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.01.627.003 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.01.627.021 I slot operator(): id 0 | task 0 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 28 +0.01.627.031 I slot operator(): id 0 | task 0 | cached n_tokens = 0, memory_seq_rm [0, end) +0.01.636.680 I slot operator(): id 0 | task 0 | cached n_tokens = 10, memory_seq_rm [10, end) +0.01.669.278 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.01.671.872 I slot operator(): id 0 | task 0 | cached n_tokens = 24, memory_seq_rm [24, end) +0.01.671.895 I slot init_sampler: id 0 | task 0 | init sampler, took 0.00 ms, tokens: text = 28, total = 28 +0.01.707.800 I slot create_check: id 0 | task 0 | created context checkpoint 2 of 32 (pos_min = 0, pos_max = 23, n_tokens = 24, size = 0.985 MiB) +0.01.746.066 I slot print_timing: id 0 | task 0 | prompt eval time = 102.36 ms / 28 tokens ( 3.66 ms per token, 273.54 tokens per second) +0.01.746.072 I slot print_timing: id 0 | task 0 | eval time = 16.67 ms / 2 tokens ( 16.67 ms per token, 59.99 tokens per second) +0.01.746.074 I slot print_timing: id 0 | task 0 | total time = 119.03 ms / 30 tokens +0.01.746.074 I slot print_timing: id 0 | task 0 | graphs reused = 1 +0.01.746.100 I slot release: id 0 | task 0 | stop processing: n_tokens = 29, truncated = 0 +0.01.746.107 I srv update_slots: all slots are idle +0.01.783.564 I srv server_strea: conv_id= (empty=1) +0.01.784.990 I srv operator(): chat format: peg-native +0.01.785.077 I slot get_availabl: id 0 | task -1 | - checking sim = 0.005 (12/2463) > 0.100 +0.01.785.081 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = 1594339 +0.01.785.273 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.01.785.286 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.01.785.287 I slot launch_slot_: id 0 | task 5 | processing task, is_child = 0 +0.01.785.297 I slot operator(): id 0 | task 5 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.01.785.301 I slot operator(): id 0 | task 5 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.01.785.303 I slot operator(): id 0 | task 5 | erased invalidated context checkpoint (pos_min = 0, pos_max = 23, n_tokens = 24, n_swa = 512, pos_next = 0, size = 0.985 MiB) +0.01.785.305 I slot operator(): id 0 | task 5 | cached n_tokens = 0, memory_seq_rm [0, end) +0.01.788.571 I slot operator(): id 0 | task 5 | cached n_tokens = 10, memory_seq_rm [10, end) +0.01.822.166 I slot create_check: id 0 | task 5 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.02.850.637 I slot operator(): id 0 | task 5 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.03.265.906 I slot create_check: id 0 | task 5 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.03.273.258 I slot operator(): id 0 | task 5 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.03.273.643 I slot init_sampler: id 0 | task 5 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.03.758.278 I slot create_check: id 0 | task 5 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.03.858.505 I slot print_timing: id 0 | task 5 | prompt eval time = 2000.78 ms / 2463 tokens ( 0.81 ms per token, 1231.02 tokens per second) +0.03.858.517 I slot print_timing: id 0 | task 5 | eval time = 72.40 ms / 5 tokens ( 18.10 ms per token, 55.25 tokens per second) +0.03.858.519 I slot print_timing: id 0 | task 5 | total time = 2073.18 ms / 2468 tokens +0.03.858.520 I slot print_timing: id 0 | task 5 | graphs reused = 4 +0.03.858.619 I slot release: id 0 | task 5 | stop processing: n_tokens = 2467, truncated = 0 +0.03.858.644 I srv update_slots: all slots are idle +0.03.902.212 I srv server_strea: conv_id= (empty=1) +0.03.903.738 I srv operator(): chat format: peg-native +0.03.903.885 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.03.903.894 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.03.904.183 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.03.904.204 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.03.904.207 I slot launch_slot_: id 0 | task 14 | processing task, is_child = 0 +0.03.904.220 I slot operator(): id 0 | task 14 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.03.904.224 I slot operator(): id 0 | task 14 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.03.904.229 I slot operator(): id 0 | task 14 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.03.905.238 I slot operator(): id 0 | task 14 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.03.906.355 I slot operator(): id 0 | task 14 | cached n_tokens = 0, memory_seq_rm [0, end) +0.03.913.120 I slot operator(): id 0 | task 14 | cached n_tokens = 10, memory_seq_rm [10, end) +0.03.947.627 I slot create_check: id 0 | task 14 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.04.948.656 I slot operator(): id 0 | task 14 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.05.364.019 I slot create_check: id 0 | task 14 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.05.372.969 I slot operator(): id 0 | task 14 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.05.373.361 I slot init_sampler: id 0 | task 14 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.05.854.215 I slot create_check: id 0 | task 14 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.05.958.863 I slot print_timing: id 0 | task 14 | prompt eval time = 1978.71 ms / 2463 tokens ( 0.80 ms per token, 1244.75 tokens per second) +0.05.958.881 I slot print_timing: id 0 | task 14 | eval time = 75.91 ms / 5 tokens ( 18.98 ms per token, 52.70 tokens per second) +0.05.958.883 I slot print_timing: id 0 | task 14 | total time = 2054.62 ms / 2468 tokens +0.05.958.885 I slot print_timing: id 0 | task 14 | graphs reused = 7 +0.05.958.987 I slot release: id 0 | task 14 | stop processing: n_tokens = 2467, truncated = 0 +0.05.959.020 I srv update_slots: all slots are idle +0.05.976.245 I srv server_strea: conv_id= (empty=1) +0.05.977.721 I srv operator(): chat format: peg-native +0.05.977.859 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.05.977.871 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.05.978.203 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.05.978.223 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.05.978.227 I slot launch_slot_: id 0 | task 23 | processing task, is_child = 0 +0.05.978.239 I slot operator(): id 0 | task 23 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.05.978.244 I slot operator(): id 0 | task 23 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.05.978.248 I slot operator(): id 0 | task 23 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.05.979.483 I slot operator(): id 0 | task 23 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.05.980.522 I slot operator(): id 0 | task 23 | cached n_tokens = 0, memory_seq_rm [0, end) +0.05.986.019 I slot operator(): id 0 | task 23 | cached n_tokens = 10, memory_seq_rm [10, end) +0.06.020.285 I slot create_check: id 0 | task 23 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.07.028.999 I slot operator(): id 0 | task 23 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.07.441.837 I slot create_check: id 0 | task 23 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.07.449.859 I slot operator(): id 0 | task 23 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.07.450.233 I slot init_sampler: id 0 | task 23 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.07.927.081 I slot create_check: id 0 | task 23 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.08.025.726 I slot print_timing: id 0 | task 23 | prompt eval time = 1975.83 ms / 2463 tokens ( 0.80 ms per token, 1246.57 tokens per second) +0.08.025.736 I slot print_timing: id 0 | task 23 | eval time = 71.64 ms / 5 tokens ( 17.91 ms per token, 55.83 tokens per second) +0.08.025.738 I slot print_timing: id 0 | task 23 | total time = 2047.47 ms / 2468 tokens +0.08.025.739 I slot print_timing: id 0 | task 23 | graphs reused = 10 +0.08.025.826 I slot release: id 0 | task 23 | stop processing: n_tokens = 2467, truncated = 0 +0.08.025.853 I srv update_slots: all slots are idle +0.08.046.340 I srv server_strea: conv_id= (empty=1) +0.08.048.629 I srv operator(): chat format: peg-native +0.08.048.796 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.08.048.804 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.08.049.073 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.08.049.090 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.08.049.093 I slot launch_slot_: id 0 | task 32 | processing task, is_child = 0 +0.08.049.102 I slot operator(): id 0 | task 32 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.08.049.105 I slot operator(): id 0 | task 32 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.08.049.109 I slot operator(): id 0 | task 32 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.08.049.962 I slot operator(): id 0 | task 32 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.08.051.125 I slot operator(): id 0 | task 32 | cached n_tokens = 0, memory_seq_rm [0, end) +0.08.057.013 I slot operator(): id 0 | task 32 | cached n_tokens = 10, memory_seq_rm [10, end) +0.08.092.466 I slot create_check: id 0 | task 32 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.09.088.687 I slot operator(): id 0 | task 32 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.09.508.225 I slot create_check: id 0 | task 32 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.09.516.745 I slot operator(): id 0 | task 32 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.09.517.120 I slot init_sampler: id 0 | task 32 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.10.000.138 I slot create_check: id 0 | task 32 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.10.102.112 I slot print_timing: id 0 | task 32 | prompt eval time = 1977.09 ms / 2463 tokens ( 0.80 ms per token, 1245.77 tokens per second) +0.10.102.127 I slot print_timing: id 0 | task 32 | eval time = 75.90 ms / 5 tokens ( 18.98 ms per token, 52.70 tokens per second) +0.10.102.129 I slot print_timing: id 0 | task 32 | total time = 2052.99 ms / 2468 tokens +0.10.102.130 I slot print_timing: id 0 | task 32 | graphs reused = 13 +0.10.102.222 I slot release: id 0 | task 32 | stop processing: n_tokens = 2467, truncated = 0 +0.10.102.252 I srv update_slots: all slots are idle +0.10.146.643 I srv server_strea: conv_id= (empty=1) +0.10.148.108 I srv operator(): chat format: peg-native +0.10.148.244 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.10.148.262 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.10.148.592 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.10.148.620 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.10.148.625 I slot launch_slot_: id 0 | task 41 | processing task, is_child = 0 +0.10.148.640 I slot operator(): id 0 | task 41 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.10.148.645 I slot operator(): id 0 | task 41 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.10.148.651 I slot operator(): id 0 | task 41 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.10.149.882 I slot operator(): id 0 | task 41 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.10.150.731 I slot operator(): id 0 | task 41 | cached n_tokens = 0, memory_seq_rm [0, end) +0.10.156.212 I slot operator(): id 0 | task 41 | cached n_tokens = 10, memory_seq_rm [10, end) +0.10.190.765 I slot create_check: id 0 | task 41 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.11.207.534 I slot operator(): id 0 | task 41 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.11.631.402 I slot create_check: id 0 | task 41 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.11.640.659 I slot operator(): id 0 | task 41 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.11.641.067 I slot init_sampler: id 0 | task 41 | init sampler, took 0.37 ms, tokens: text = 2463, total = 2463 +0.12.118.124 I slot create_check: id 0 | task 41 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.12.219.148 I slot print_timing: id 0 | task 41 | prompt eval time = 1996.22 ms / 2463 tokens ( 0.81 ms per token, 1233.83 tokens per second) +0.12.219.160 I slot print_timing: id 0 | task 41 | eval time = 74.27 ms / 5 tokens ( 18.57 ms per token, 53.86 tokens per second) +0.12.219.162 I slot print_timing: id 0 | task 41 | total time = 2070.49 ms / 2468 tokens +0.12.219.163 I slot print_timing: id 0 | task 41 | graphs reused = 16 +0.12.219.249 I slot release: id 0 | task 41 | stop processing: n_tokens = 2467, truncated = 0 +0.12.219.285 I srv update_slots: all slots are idle +0.12.267.047 I srv server_strea: conv_id= (empty=1) +0.12.269.818 I srv operator(): chat format: peg-native +0.12.270.075 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.12.270.085 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.12.270.409 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.12.270.430 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.12.270.433 I slot launch_slot_: id 0 | task 50 | processing task, is_child = 0 +0.12.270.444 I slot operator(): id 0 | task 50 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.12.270.450 I slot operator(): id 0 | task 50 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.12.270.454 I slot operator(): id 0 | task 50 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.12.271.572 I slot operator(): id 0 | task 50 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.12.272.471 I slot operator(): id 0 | task 50 | cached n_tokens = 0, memory_seq_rm [0, end) +0.12.278.695 I slot operator(): id 0 | task 50 | cached n_tokens = 10, memory_seq_rm [10, end) +0.12.315.817 I slot create_check: id 0 | task 50 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.13.319.555 I slot operator(): id 0 | task 50 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.13.734.494 I slot create_check: id 0 | task 50 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.13.744.105 I slot operator(): id 0 | task 50 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.13.744.487 I slot init_sampler: id 0 | task 50 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.14.222.321 I slot create_check: id 0 | task 50 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.14.321.302 I slot print_timing: id 0 | task 50 | prompt eval time = 1978.45 ms / 2463 tokens ( 0.80 ms per token, 1244.92 tokens per second) +0.14.321.312 I slot print_timing: id 0 | task 50 | eval time = 72.39 ms / 5 tokens ( 18.10 ms per token, 55.25 tokens per second) +0.14.321.314 I slot print_timing: id 0 | task 50 | total time = 2050.84 ms / 2468 tokens +0.14.321.315 I slot print_timing: id 0 | task 50 | graphs reused = 19 +0.14.321.403 I slot release: id 0 | task 50 | stop processing: n_tokens = 2467, truncated = 0 +0.14.321.430 I srv update_slots: all slots are idle +0.14.363.208 I srv server_strea: conv_id= (empty=1) +0.14.364.704 I srv operator(): chat format: peg-native +0.14.364.835 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.14.364.845 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.14.365.131 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.14.365.150 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.14.365.153 I slot launch_slot_: id 0 | task 59 | processing task, is_child = 0 +0.14.365.165 I slot operator(): id 0 | task 59 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.14.365.170 I slot operator(): id 0 | task 59 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.14.365.174 I slot operator(): id 0 | task 59 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.14.366.142 I slot operator(): id 0 | task 59 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.14.367.087 I slot operator(): id 0 | task 59 | cached n_tokens = 0, memory_seq_rm [0, end) +0.14.370.903 I slot operator(): id 0 | task 59 | cached n_tokens = 10, memory_seq_rm [10, end) +0.14.407.467 I slot create_check: id 0 | task 59 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.15.403.305 I slot operator(): id 0 | task 59 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.15.821.183 I slot create_check: id 0 | task 59 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.15.829.454 I slot operator(): id 0 | task 59 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.15.829.826 I slot init_sampler: id 0 | task 59 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.16.305.706 I slot create_check: id 0 | task 59 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.16.404.865 I slot print_timing: id 0 | task 59 | prompt eval time = 1967.37 ms / 2463 tokens ( 0.80 ms per token, 1251.92 tokens per second) +0.16.404.878 I slot print_timing: id 0 | task 59 | eval time = 72.31 ms / 5 tokens ( 18.08 ms per token, 55.32 tokens per second) +0.16.404.880 I slot print_timing: id 0 | task 59 | total time = 2039.68 ms / 2468 tokens +0.16.404.881 I slot print_timing: id 0 | task 59 | graphs reused = 22 +0.16.404.977 I slot release: id 0 | task 59 | stop processing: n_tokens = 2467, truncated = 0 +0.16.405.006 I srv update_slots: all slots are idle +0.16.449.471 I srv server_strea: conv_id= (empty=1) +0.16.451.827 I srv operator(): chat format: peg-native +0.16.452.065 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.16.452.082 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.16.452.420 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.16.452.444 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.16.452.447 I slot launch_slot_: id 0 | task 68 | processing task, is_child = 0 +0.16.452.457 I slot operator(): id 0 | task 68 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.16.452.461 I slot operator(): id 0 | task 68 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.16.452.464 I slot operator(): id 0 | task 68 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.16.453.337 I slot operator(): id 0 | task 68 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.16.454.225 I slot operator(): id 0 | task 68 | cached n_tokens = 0, memory_seq_rm [0, end) +0.16.457.789 I slot operator(): id 0 | task 68 | cached n_tokens = 10, memory_seq_rm [10, end) +0.16.497.114 I slot create_check: id 0 | task 68 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.17.501.463 I slot operator(): id 0 | task 68 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.17.914.712 I slot create_check: id 0 | task 68 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.17.921.236 I slot operator(): id 0 | task 68 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.17.921.603 I slot init_sampler: id 0 | task 68 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.18.400.198 I slot create_check: id 0 | task 68 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.18.499.443 I slot print_timing: id 0 | task 68 | prompt eval time = 1973.21 ms / 2463 tokens ( 0.80 ms per token, 1248.22 tokens per second) +0.18.499.453 I slot print_timing: id 0 | task 68 | eval time = 73.76 ms / 5 tokens ( 18.44 ms per token, 54.23 tokens per second) +0.18.499.455 I slot print_timing: id 0 | task 68 | total time = 2046.97 ms / 2468 tokens +0.18.499.456 I slot print_timing: id 0 | task 68 | graphs reused = 25 +0.18.499.540 I slot release: id 0 | task 68 | stop processing: n_tokens = 2467, truncated = 0 +0.18.499.561 I srv update_slots: all slots are idle +0.18.527.023 I srv server_strea: conv_id= (empty=1) +0.18.528.501 I srv operator(): chat format: peg-native +0.18.528.626 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.18.528.636 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.18.528.943 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.18.528.963 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.18.528.968 I slot launch_slot_: id 0 | task 77 | processing task, is_child = 0 +0.18.528.981 I slot operator(): id 0 | task 77 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.18.528.986 I slot operator(): id 0 | task 77 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.18.528.990 I slot operator(): id 0 | task 77 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.18.529.864 I slot operator(): id 0 | task 77 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.18.530.984 I slot operator(): id 0 | task 77 | cached n_tokens = 0, memory_seq_rm [0, end) +0.18.536.338 I slot operator(): id 0 | task 77 | cached n_tokens = 10, memory_seq_rm [10, end) +0.18.568.775 I slot create_check: id 0 | task 77 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.19.566.136 I slot operator(): id 0 | task 77 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.19.986.109 I slot create_check: id 0 | task 77 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.19.995.990 I slot operator(): id 0 | task 77 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.19.996.368 I slot init_sampler: id 0 | task 77 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.20.471.632 I slot create_check: id 0 | task 77 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.20.572.120 I slot print_timing: id 0 | task 77 | prompt eval time = 1970.27 ms / 2463 tokens ( 0.80 ms per token, 1250.08 tokens per second) +0.20.572.131 I slot print_timing: id 0 | task 77 | eval time = 72.85 ms / 5 tokens ( 18.21 ms per token, 54.91 tokens per second) +0.20.572.133 I slot print_timing: id 0 | task 77 | total time = 2043.12 ms / 2468 tokens +0.20.572.134 I slot print_timing: id 0 | task 77 | graphs reused = 28 +0.20.572.227 I slot release: id 0 | task 77 | stop processing: n_tokens = 2467, truncated = 0 +0.20.572.253 I srv update_slots: all slots are idle +0.20.631.074 I srv server_strea: conv_id= (empty=1) +0.20.634.833 I srv operator(): chat format: peg-native +0.20.635.070 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.20.635.079 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.20.635.333 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.20.635.357 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.20.635.360 I slot launch_slot_: id 0 | task 86 | processing task, is_child = 0 +0.20.635.371 I slot operator(): id 0 | task 86 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.20.635.375 I slot operator(): id 0 | task 86 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.20.635.379 I slot operator(): id 0 | task 86 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.20.636.320 I slot operator(): id 0 | task 86 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.20.637.259 I slot operator(): id 0 | task 86 | cached n_tokens = 0, memory_seq_rm [0, end) +0.20.640.983 I slot operator(): id 0 | task 86 | cached n_tokens = 10, memory_seq_rm [10, end) +0.20.679.181 I slot create_check: id 0 | task 86 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.21.697.096 I slot operator(): id 0 | task 86 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.22.134.484 I slot create_check: id 0 | task 86 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.22.143.793 I slot operator(): id 0 | task 86 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.22.144.215 I slot init_sampler: id 0 | task 86 | init sampler, took 0.38 ms, tokens: text = 2463, total = 2463 +0.22.626.486 I slot create_check: id 0 | task 86 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.22.734.940 I slot print_timing: id 0 | task 86 | prompt eval time = 2021.38 ms / 2463 tokens ( 0.82 ms per token, 1218.48 tokens per second) +0.22.734.953 I slot print_timing: id 0 | task 86 | eval time = 78.17 ms / 5 tokens ( 19.54 ms per token, 51.17 tokens per second) +0.22.734.955 I slot print_timing: id 0 | task 86 | total time = 2099.55 ms / 2468 tokens +0.22.734.956 I slot print_timing: id 0 | task 86 | graphs reused = 31 +0.22.735.048 I slot release: id 0 | task 86 | stop processing: n_tokens = 2467, truncated = 0 +0.22.735.076 I srv update_slots: all slots are idle +0.22.801.946 I srv server_strea: conv_id= (empty=1) +0.22.804.511 I srv operator(): chat format: peg-native +0.22.804.686 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.22.804.692 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.22.804.937 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.22.804.954 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.22.804.957 I slot launch_slot_: id 0 | task 95 | processing task, is_child = 0 +0.22.804.968 I slot operator(): id 0 | task 95 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.22.804.972 I slot operator(): id 0 | task 95 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.22.804.975 I slot operator(): id 0 | task 95 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.22.805.897 I slot operator(): id 0 | task 95 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.22.806.858 I slot operator(): id 0 | task 95 | cached n_tokens = 0, memory_seq_rm [0, end) +0.22.812.102 I slot operator(): id 0 | task 95 | cached n_tokens = 10, memory_seq_rm [10, end) +0.22.846.361 I slot create_check: id 0 | task 95 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.23.858.632 I slot operator(): id 0 | task 95 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.24.271.576 I slot create_check: id 0 | task 95 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.24.278.174 I slot operator(): id 0 | task 95 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.24.278.546 I slot init_sampler: id 0 | task 95 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.24.757.637 I slot create_check: id 0 | task 95 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.24.859.661 I slot print_timing: id 0 | task 95 | prompt eval time = 1978.68 ms / 2463 tokens ( 0.80 ms per token, 1244.77 tokens per second) +0.24.859.674 I slot print_timing: id 0 | task 95 | eval time = 75.97 ms / 5 tokens ( 18.99 ms per token, 52.65 tokens per second) +0.24.859.676 I slot print_timing: id 0 | task 95 | total time = 2054.65 ms / 2468 tokens +0.24.859.677 I slot print_timing: id 0 | task 95 | graphs reused = 34 +0.24.859.777 I slot release: id 0 | task 95 | stop processing: n_tokens = 2467, truncated = 0 +0.24.859.806 I srv update_slots: all slots are idle +0.24.916.596 I srv server_strea: conv_id= (empty=1) +0.24.918.095 I srv operator(): chat format: peg-native +0.24.918.223 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.24.918.231 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.24.918.473 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.24.918.498 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.24.918.501 I slot launch_slot_: id 0 | task 104 | processing task, is_child = 0 +0.24.918.514 I slot operator(): id 0 | task 104 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.24.918.518 I slot operator(): id 0 | task 104 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.24.918.522 I slot operator(): id 0 | task 104 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.24.919.372 I slot operator(): id 0 | task 104 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.24.920.256 I slot operator(): id 0 | task 104 | cached n_tokens = 0, memory_seq_rm [0, end) +0.24.925.359 I slot operator(): id 0 | task 104 | cached n_tokens = 10, memory_seq_rm [10, end) +0.24.961.868 I slot create_check: id 0 | task 104 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.25.976.482 I slot operator(): id 0 | task 104 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.26.390.676 I slot create_check: id 0 | task 104 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.26.397.400 I slot operator(): id 0 | task 104 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.26.397.773 I slot init_sampler: id 0 | task 104 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.26.869.338 I slot create_check: id 0 | task 104 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.26.968.428 I slot print_timing: id 0 | task 104 | prompt eval time = 1978.39 ms / 2463 tokens ( 0.80 ms per token, 1244.95 tokens per second) +0.26.968.439 I slot print_timing: id 0 | task 104 | eval time = 71.51 ms / 5 tokens ( 17.88 ms per token, 55.94 tokens per second) +0.26.968.441 I slot print_timing: id 0 | task 104 | total time = 2049.90 ms / 2468 tokens +0.26.968.442 I slot print_timing: id 0 | task 104 | graphs reused = 37 +0.26.968.536 I slot release: id 0 | task 104 | stop processing: n_tokens = 2467, truncated = 0 +0.26.968.560 I srv update_slots: all slots are idle +0.27.000.657 I srv server_strea: conv_id= (empty=1) +0.27.002.123 I srv operator(): chat format: peg-native +0.27.002.299 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.27.002.345 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.27.002.742 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.27.002.771 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.27.002.777 I slot launch_slot_: id 0 | task 113 | processing task, is_child = 0 +0.27.002.797 I slot operator(): id 0 | task 113 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.27.002.803 I slot operator(): id 0 | task 113 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.27.002.809 I slot operator(): id 0 | task 113 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.27.004.039 I slot operator(): id 0 | task 113 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.27.005.070 I slot operator(): id 0 | task 113 | cached n_tokens = 0, memory_seq_rm [0, end) +0.27.010.888 I slot operator(): id 0 | task 113 | cached n_tokens = 10, memory_seq_rm [10, end) +0.27.045.138 I slot create_check: id 0 | task 113 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.28.044.653 I slot operator(): id 0 | task 113 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.28.461.696 I slot create_check: id 0 | task 113 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.28.468.418 I slot operator(): id 0 | task 113 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.28.468.786 I slot init_sampler: id 0 | task 113 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.28.946.766 I slot create_check: id 0 | task 113 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.29.048.132 I slot print_timing: id 0 | task 113 | prompt eval time = 1970.85 ms / 2463 tokens ( 0.80 ms per token, 1249.72 tokens per second) +0.29.048.150 I slot print_timing: id 0 | task 113 | eval time = 74.46 ms / 5 tokens ( 18.62 ms per token, 53.72 tokens per second) +0.29.048.152 I slot print_timing: id 0 | task 113 | total time = 2045.31 ms / 2468 tokens +0.29.048.154 I slot print_timing: id 0 | task 113 | graphs reused = 40 +0.29.048.256 I slot release: id 0 | task 113 | stop processing: n_tokens = 2467, truncated = 0 +0.29.048.287 I srv update_slots: all slots are idle +0.29.109.708 I srv server_strea: conv_id= (empty=1) +0.29.112.111 I srv operator(): chat format: peg-native +0.29.112.275 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.29.112.281 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.29.112.525 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.29.112.549 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.29.112.553 I slot launch_slot_: id 0 | task 122 | processing task, is_child = 0 +0.29.112.564 I slot operator(): id 0 | task 122 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.29.112.568 I slot operator(): id 0 | task 122 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.29.112.572 I slot operator(): id 0 | task 122 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.29.113.512 I slot operator(): id 0 | task 122 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.29.114.414 I slot operator(): id 0 | task 122 | cached n_tokens = 0, memory_seq_rm [0, end) +0.29.117.935 I slot operator(): id 0 | task 122 | cached n_tokens = 10, memory_seq_rm [10, end) +0.29.153.961 I slot create_check: id 0 | task 122 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.30.176.659 I slot operator(): id 0 | task 122 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.30.598.627 I slot create_check: id 0 | task 122 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.30.605.355 I slot operator(): id 0 | task 122 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.30.605.730 I slot init_sampler: id 0 | task 122 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.31.081.620 I slot create_check: id 0 | task 122 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.31.182.952 I slot print_timing: id 0 | task 122 | prompt eval time = 1995.27 ms / 2463 tokens ( 0.81 ms per token, 1234.42 tokens per second) +0.31.182.962 I slot print_timing: id 0 | task 122 | eval time = 75.09 ms / 5 tokens ( 18.77 ms per token, 53.27 tokens per second) +0.31.182.964 I slot print_timing: id 0 | task 122 | total time = 2070.36 ms / 2468 tokens +0.31.182.966 I slot print_timing: id 0 | task 122 | graphs reused = 43 +0.31.183.050 I slot release: id 0 | task 122 | stop processing: n_tokens = 2467, truncated = 0 +0.31.183.079 I srv update_slots: all slots are idle +0.31.239.058 I srv server_strea: conv_id= (empty=1) +0.31.240.563 I srv operator(): chat format: peg-native +0.31.240.741 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.31.240.759 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.31.241.145 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.31.241.176 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.31.241.181 I slot launch_slot_: id 0 | task 131 | processing task, is_child = 0 +0.31.241.194 I slot operator(): id 0 | task 131 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.31.241.198 I slot operator(): id 0 | task 131 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.31.241.203 I slot operator(): id 0 | task 131 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.31.242.443 I slot operator(): id 0 | task 131 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.31.243.400 I slot operator(): id 0 | task 131 | cached n_tokens = 0, memory_seq_rm [0, end) +0.31.248.238 I slot operator(): id 0 | task 131 | cached n_tokens = 10, memory_seq_rm [10, end) +0.31.284.261 I slot create_check: id 0 | task 131 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.32.302.678 I slot operator(): id 0 | task 131 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.32.719.605 I slot create_check: id 0 | task 131 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.32.730.472 I slot operator(): id 0 | task 131 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.32.730.920 I slot init_sampler: id 0 | task 131 | init sampler, took 0.40 ms, tokens: text = 2463, total = 2463 +0.33.211.537 I slot create_check: id 0 | task 131 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.33.318.307 I slot print_timing: id 0 | task 131 | prompt eval time = 1999.62 ms / 2463 tokens ( 0.81 ms per token, 1231.74 tokens per second) +0.33.318.317 I slot print_timing: id 0 | task 131 | eval time = 77.47 ms / 5 tokens ( 19.37 ms per token, 51.63 tokens per second) +0.33.318.319 I slot print_timing: id 0 | task 131 | total time = 2077.09 ms / 2468 tokens +0.33.318.320 I slot print_timing: id 0 | task 131 | graphs reused = 46 +0.33.318.410 I slot release: id 0 | task 131 | stop processing: n_tokens = 2467, truncated = 0 +0.33.318.439 I srv update_slots: all slots are idle +0.33.359.157 I srv server_strea: conv_id= (empty=1) +0.33.361.975 I srv operator(): chat format: peg-native +0.33.362.228 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.33.362.244 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.33.362.553 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.33.362.581 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.33.362.586 I slot launch_slot_: id 0 | task 140 | processing task, is_child = 0 +0.33.362.599 I slot operator(): id 0 | task 140 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.33.362.605 I slot operator(): id 0 | task 140 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.33.362.612 I slot operator(): id 0 | task 140 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.33.363.631 I slot operator(): id 0 | task 140 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.33.364.502 I slot operator(): id 0 | task 140 | cached n_tokens = 0, memory_seq_rm [0, end) +0.33.368.383 I slot operator(): id 0 | task 140 | cached n_tokens = 10, memory_seq_rm [10, end) +0.33.404.126 I slot create_check: id 0 | task 140 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.34.405.257 I slot operator(): id 0 | task 140 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.34.824.941 I slot create_check: id 0 | task 140 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.34.832.790 I slot operator(): id 0 | task 140 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.34.833.407 I slot init_sampler: id 0 | task 140 | init sampler, took 0.56 ms, tokens: text = 2463, total = 2463 +0.35.313.504 I slot create_check: id 0 | task 140 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.35.416.401 I slot print_timing: id 0 | task 140 | prompt eval time = 1978.47 ms / 2463 tokens ( 0.80 ms per token, 1244.90 tokens per second) +0.35.416.420 I slot print_timing: id 0 | task 140 | eval time = 75.31 ms / 5 tokens ( 18.83 ms per token, 53.11 tokens per second) +0.35.416.422 I slot print_timing: id 0 | task 140 | total time = 2053.78 ms / 2468 tokens +0.35.416.423 I slot print_timing: id 0 | task 140 | graphs reused = 49 +0.35.416.542 I slot release: id 0 | task 140 | stop processing: n_tokens = 2467, truncated = 0 +0.35.416.569 I srv update_slots: all slots are idle +0.35.471.423 I srv server_strea: conv_id= (empty=1) +0.35.474.755 I srv operator(): chat format: peg-native +0.35.475.065 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.35.475.075 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.35.475.367 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.35.475.393 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.35.475.397 I slot launch_slot_: id 0 | task 149 | processing task, is_child = 0 +0.35.475.409 I slot operator(): id 0 | task 149 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.35.475.414 I slot operator(): id 0 | task 149 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.35.475.421 I slot operator(): id 0 | task 149 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.35.476.781 I slot operator(): id 0 | task 149 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.35.477.998 I slot operator(): id 0 | task 149 | cached n_tokens = 0, memory_seq_rm [0, end) +0.35.483.795 I slot operator(): id 0 | task 149 | cached n_tokens = 10, memory_seq_rm [10, end) +0.35.518.929 I slot create_check: id 0 | task 149 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.36.540.789 I slot operator(): id 0 | task 149 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.36.957.626 I slot create_check: id 0 | task 149 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.36.965.426 I slot operator(): id 0 | task 149 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.36.965.873 I slot init_sampler: id 0 | task 149 | init sampler, took 0.41 ms, tokens: text = 2463, total = 2463 +0.37.442.731 I slot create_check: id 0 | task 149 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.37.546.313 I slot print_timing: id 0 | task 149 | prompt eval time = 1996.78 ms / 2463 tokens ( 0.81 ms per token, 1233.49 tokens per second) +0.37.546.327 I slot print_timing: id 0 | task 149 | eval time = 74.10 ms / 5 tokens ( 18.53 ms per token, 53.98 tokens per second) +0.37.546.329 I slot print_timing: id 0 | task 149 | total time = 2070.88 ms / 2468 tokens +0.37.546.330 I slot print_timing: id 0 | task 149 | graphs reused = 52 +0.37.546.413 I slot release: id 0 | task 149 | stop processing: n_tokens = 2467, truncated = 0 +0.37.546.441 I srv update_slots: all slots are idle +0.37.591.103 I srv server_strea: conv_id= (empty=1) +0.37.593.523 I srv operator(): chat format: peg-native +0.37.593.743 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.37.593.756 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.37.594.135 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.37.594.164 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.37.594.169 I slot launch_slot_: id 0 | task 158 | processing task, is_child = 0 +0.37.594.182 I slot operator(): id 0 | task 158 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.37.594.188 I slot operator(): id 0 | task 158 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.37.594.193 I slot operator(): id 0 | task 158 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.37.595.426 I slot operator(): id 0 | task 158 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.37.596.285 I slot operator(): id 0 | task 158 | cached n_tokens = 0, memory_seq_rm [0, end) +0.37.601.020 I slot operator(): id 0 | task 158 | cached n_tokens = 10, memory_seq_rm [10, end) +0.37.637.226 I slot create_check: id 0 | task 158 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.38.637.501 I slot operator(): id 0 | task 158 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.39.048.198 I slot create_check: id 0 | task 158 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.39.057.305 I slot operator(): id 0 | task 158 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.39.057.681 I slot init_sampler: id 0 | task 158 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.39.531.552 I slot create_check: id 0 | task 158 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.39.631.927 I slot print_timing: id 0 | task 158 | prompt eval time = 1964.50 ms / 2463 tokens ( 0.80 ms per token, 1253.75 tokens per second) +0.39.631.939 I slot print_timing: id 0 | task 158 | eval time = 73.23 ms / 5 tokens ( 18.31 ms per token, 54.62 tokens per second) +0.39.631.941 I slot print_timing: id 0 | task 158 | total time = 2037.73 ms / 2468 tokens +0.39.631.942 I slot print_timing: id 0 | task 158 | graphs reused = 55 +0.39.632.030 I slot release: id 0 | task 158 | stop processing: n_tokens = 2467, truncated = 0 +0.39.632.055 I srv update_slots: all slots are idle +0.39.672.494 I srv server_strea: conv_id= (empty=1) +0.39.673.966 I srv operator(): chat format: peg-native +0.39.674.108 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.39.674.122 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.39.674.525 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.39.674.561 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.39.674.567 I slot launch_slot_: id 0 | task 167 | processing task, is_child = 0 +0.39.674.584 I slot operator(): id 0 | task 167 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.39.674.590 I slot operator(): id 0 | task 167 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.39.674.597 I slot operator(): id 0 | task 167 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.39.675.647 I slot operator(): id 0 | task 167 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.39.676.569 I slot operator(): id 0 | task 167 | cached n_tokens = 0, memory_seq_rm [0, end) +0.39.680.971 I slot operator(): id 0 | task 167 | cached n_tokens = 10, memory_seq_rm [10, end) +0.39.716.757 I slot create_check: id 0 | task 167 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.40.713.219 I slot operator(): id 0 | task 167 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.41.126.091 I slot create_check: id 0 | task 167 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.41.134.637 I slot operator(): id 0 | task 167 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.41.135.024 I slot init_sampler: id 0 | task 167 | init sampler, took 0.36 ms, tokens: text = 2463, total = 2463 +0.41.614.090 I slot create_check: id 0 | task 167 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.41.727.082 I slot print_timing: id 0 | task 167 | prompt eval time = 1970.03 ms / 2463 tokens ( 0.80 ms per token, 1250.24 tokens per second) +0.41.727.098 I slot print_timing: id 0 | task 167 | eval time = 82.44 ms / 5 tokens ( 20.61 ms per token, 48.52 tokens per second) +0.41.727.101 I slot print_timing: id 0 | task 167 | total time = 2052.47 ms / 2468 tokens +0.41.727.105 I slot print_timing: id 0 | task 167 | graphs reused = 58 +0.41.727.199 I slot release: id 0 | task 167 | stop processing: n_tokens = 2467, truncated = 0 +0.41.727.224 I srv update_slots: all slots are idle +0.41.761.267 I srv server_strea: conv_id= (empty=1) +0.41.764.500 I srv operator(): chat format: peg-native +0.41.764.731 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.41.764.741 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.41.765.027 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.41.765.045 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.41.765.048 I slot launch_slot_: id 0 | task 176 | processing task, is_child = 0 +0.41.765.057 I slot operator(): id 0 | task 176 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.41.765.061 I slot operator(): id 0 | task 176 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.41.765.065 I slot operator(): id 0 | task 176 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.41.766.123 I slot operator(): id 0 | task 176 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.41.767.208 I slot operator(): id 0 | task 176 | cached n_tokens = 0, memory_seq_rm [0, end) +0.41.772.545 I slot operator(): id 0 | task 176 | cached n_tokens = 10, memory_seq_rm [10, end) +0.41.810.364 I slot create_check: id 0 | task 176 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.42.842.226 I slot operator(): id 0 | task 176 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.43.259.509 I slot create_check: id 0 | task 176 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.43.269.571 I slot operator(): id 0 | task 176 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.43.270.156 I slot init_sampler: id 0 | task 176 | init sampler, took 0.54 ms, tokens: text = 2463, total = 2463 +0.43.746.431 I slot create_check: id 0 | task 176 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.43.849.814 I slot print_timing: id 0 | task 176 | prompt eval time = 2009.16 ms / 2463 tokens ( 0.82 ms per token, 1225.89 tokens per second) +0.43.849.827 I slot print_timing: id 0 | task 176 | eval time = 75.58 ms / 5 tokens ( 18.89 ms per token, 52.93 tokens per second) +0.43.849.829 I slot print_timing: id 0 | task 176 | total time = 2084.73 ms / 2468 tokens +0.43.849.830 I slot print_timing: id 0 | task 176 | graphs reused = 61 +0.43.849.926 I slot release: id 0 | task 176 | stop processing: n_tokens = 2467, truncated = 0 +0.43.849.953 I srv update_slots: all slots are idle +0.43.916.655 I srv server_strea: conv_id= (empty=1) +0.43.920.254 I srv operator(): chat format: peg-native +0.43.920.474 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.43.920.487 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.43.920.840 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.43.920.866 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.43.920.875 I slot launch_slot_: id 0 | task 185 | processing task, is_child = 0 +0.43.920.894 I slot operator(): id 0 | task 185 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.43.920.903 I slot operator(): id 0 | task 185 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.43.920.910 I slot operator(): id 0 | task 185 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.43.922.079 I slot operator(): id 0 | task 185 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.43.923.103 I slot operator(): id 0 | task 185 | cached n_tokens = 0, memory_seq_rm [0, end) +0.43.928.415 I slot operator(): id 0 | task 185 | cached n_tokens = 10, memory_seq_rm [10, end) +0.43.964.348 I slot create_check: id 0 | task 185 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.44.970.488 I slot operator(): id 0 | task 185 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.45.389.595 I slot create_check: id 0 | task 185 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.45.396.586 I slot operator(): id 0 | task 185 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.45.396.958 I slot init_sampler: id 0 | task 185 | init sampler, took 0.34 ms, tokens: text = 2463, total = 2463 +0.45.878.799 I slot create_check: id 0 | task 185 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.45.982.672 I slot print_timing: id 0 | task 185 | prompt eval time = 1985.72 ms / 2463 tokens ( 0.81 ms per token, 1240.35 tokens per second) +0.45.982.687 I slot print_timing: id 0 | task 185 | eval time = 76.04 ms / 5 tokens ( 19.01 ms per token, 52.60 tokens per second) +0.45.982.688 I slot print_timing: id 0 | task 185 | total time = 2061.76 ms / 2468 tokens +0.45.982.690 I slot print_timing: id 0 | task 185 | graphs reused = 64 +0.45.982.779 I slot release: id 0 | task 185 | stop processing: n_tokens = 2467, truncated = 0 +0.45.982.804 I srv update_slots: all slots are idle +0.46.024.111 I srv server_strea: conv_id= (empty=1) +0.46.026.669 I srv operator(): chat format: peg-native +0.46.026.823 I slot get_availabl: id 0 | task -1 | - checking sim = 1.000 (2463/2463) > 0.100 +0.46.026.829 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 1.000 (> 0.100 thold), f_keep = 0.998 +0.46.027.098 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.46.027.118 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.46.027.121 I slot launch_slot_: id 0 | task 194 | processing task, is_child = 0 +0.46.027.131 I slot operator(): id 0 | task 194 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 2463 +0.46.027.135 I slot operator(): id 0 | task 194 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.46.027.138 I slot operator(): id 0 | task 194 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.46.028.090 I slot operator(): id 0 | task 194 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.46.029.075 I slot operator(): id 0 | task 194 | cached n_tokens = 0, memory_seq_rm [0, end) +0.46.033.045 I slot operator(): id 0 | task 194 | cached n_tokens = 10, memory_seq_rm [10, end) +0.46.069.168 I slot create_check: id 0 | task 194 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.47.068.261 I slot operator(): id 0 | task 194 | cached n_tokens = 1947, memory_seq_rm [1947, end) +0.47.481.195 I slot create_check: id 0 | task 194 | created context checkpoint 2 of 32 (pos_min = 923, pos_max = 1946, n_tokens = 1947, size = 21.006 MiB) +0.47.490.966 I slot operator(): id 0 | task 194 | cached n_tokens = 2459, memory_seq_rm [2459, end) +0.47.491.387 I slot init_sampler: id 0 | task 194 | init sampler, took 0.38 ms, tokens: text = 2463, total = 2463 +0.47.999.251 I slot create_check: id 0 | task 194 | created context checkpoint 3 of 32 (pos_min = 1435, pos_max = 2458, n_tokens = 2459, size = 21.006 MiB) +0.48.100.586 I slot print_timing: id 0 | task 194 | prompt eval time = 2000.37 ms / 2463 tokens ( 0.81 ms per token, 1231.27 tokens per second) +0.48.100.599 I slot print_timing: id 0 | task 194 | eval time = 73.06 ms / 5 tokens ( 18.27 ms per token, 54.75 tokens per second) +0.48.100.600 I slot print_timing: id 0 | task 194 | total time = 2073.43 ms / 2468 tokens +0.48.100.602 I slot print_timing: id 0 | task 194 | graphs reused = 67 +0.48.100.677 I slot release: id 0 | task 194 | stop processing: n_tokens = 2467, truncated = 0 +0.48.100.704 I srv update_slots: all slots are idle +0.48.167.695 I srv server_strea: conv_id= (empty=1) +0.48.167.986 I srv operator(): chat format: peg-native +0.48.168.181 I slot get_availabl: id 0 | task -1 | - checking sim = 0.429 (12/28) > 0.100 +0.48.168.196 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, f_sim_best = 0.429 (> 0.100 thold), f_keep = 0.005 +0.48.168.538 I slot launch_slot_: id 0 | task -1 | sampler chain: logits -> ?penalties -> ?dry -> ?top-n-sigma -> ?top-k -> ?typical -> top-p -> min-p -> ?xtc -> temp-ext -> dist +0.48.168.577 I slot launch_slot_: id 0 | task -1 | sampler params: + repeat_last_n = 64, repeat_penalty = 1.000, frequency_penalty = 0.000, presence_penalty = 0.000 + dry_multiplier = 0.000, dry_base = 1.750, dry_allowed_length = 2, dry_penalty_last_n = 64 + top_k = 0, top_p = 0.950, min_p = 0.050, xtc_probability = 0.000, xtc_threshold = 0.100, typical_p = 1.000, top_n_sigma = -1.000, temp = 0.000 + mirostat = 0, mirostat_lr = 0.100, mirostat_ent = 5.000, adaptive_target = -1.000, adaptive_decay = 0.900 +0.48.168.581 I slot launch_slot_: id 0 | task 203 | processing task, is_child = 0 +0.48.168.591 I slot operator(): id 0 | task 203 | new prompt, n_ctx_slot = 4096, n_keep = 0, task.n_tokens = 28 +0.48.168.596 I slot operator(): id 0 | task 203 | erased invalidated context checkpoint (pos_min = 0, pos_max = 9, n_tokens = 10, n_swa = 512, pos_next = 0, size = 0.411 MiB) +0.48.168.601 I slot operator(): id 0 | task 203 | erased invalidated context checkpoint (pos_min = 923, pos_max = 1946, n_tokens = 1947, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.48.169.751 I slot operator(): id 0 | task 203 | erased invalidated context checkpoint (pos_min = 1435, pos_max = 2458, n_tokens = 2459, n_swa = 512, pos_next = 0, size = 21.006 MiB) +0.48.170.562 I slot operator(): id 0 | task 203 | cached n_tokens = 0, memory_seq_rm [0, end) +0.48.174.194 I slot operator(): id 0 | task 203 | cached n_tokens = 10, memory_seq_rm [10, end) +0.48.210.291 I slot create_check: id 0 | task 203 | created context checkpoint 1 of 32 (pos_min = 0, pos_max = 9, n_tokens = 10, size = 0.411 MiB) +0.48.213.477 I slot operator(): id 0 | task 203 | cached n_tokens = 24, memory_seq_rm [24, end) +0.48.213.515 I slot init_sampler: id 0 | task 203 | init sampler, took 0.01 ms, tokens: text = 28, total = 28 +0.48.249.745 I slot create_check: id 0 | task 203 | created context checkpoint 2 of 32 (pos_min = 0, pos_max = 23, n_tokens = 24, size = 0.985 MiB) +0.48.290.168 I slot print_timing: id 0 | task 203 | prompt eval time = 102.46 ms / 28 tokens ( 3.66 ms per token, 273.27 tokens per second) +0.48.290.180 I slot print_timing: id 0 | task 203 | eval time = 19.09 ms / 2 tokens ( 19.09 ms per token, 52.38 tokens per second) +0.48.290.182 I slot print_timing: id 0 | task 203 | total time = 121.56 ms / 30 tokens +0.48.290.183 I slot print_timing: id 0 | task 203 | graphs reused = 67 +0.48.290.221 I slot release: id 0 | task 203 | stop processing: n_tokens = 29, truncated = 0 +0.48.290.249 I srv update_slots: all slots are idle diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_wrong_device_20260923.driver.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_wrong_device_20260923.driver.log new file mode 100644 index 000000000..7b347d621 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_wrong_device_20260923.driver.log @@ -0,0 +1,101 @@ +INFO 09-23 20:17:49 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 20:17:49 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 20:17:54 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +WARNING 09-23 20:17:54 [stage_init_utils.py:1887] [stage_init] Failed to load transfer config: [Errno 13] Permission denied: 'C:\\Users\\zhout\\w2' +ERROR 09-23 20:17:58 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 0 initialized client(s) + +ERROR 09-23 20:17:58 [stage_runtime.py:265] Traceback (most recent call last): + +ERROR 09-23 20:17:58 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize + +ERROR 09-23 20:17:58 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + +ERROR 09-23 20:17:58 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-23 20:17:58 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas + +ERROR 09-23 20:17:58 [stage_runtime.py:265] raise primary_exc + +ERROR 09-23 20:17:58 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group + +ERROR 09-23 20:17:58 [stage_runtime.py:265] client = self._initialize_replica( + +ERROR 09-23 20:17:58 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-23 20:17:58 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 817, in _initialize_replica + +ERROR 09-23 20:17:58 [stage_runtime.py:265] return create_graph_client( + +ERROR 09-23 20:17:58 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-23 20:17:58 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\backends\__init__.py", line 15, in create_graph_client + +ERROR 09-23 20:17:58 [stage_runtime.py:265] return LlamaCppTextStageClient(metadata, backend, ledger, reservation) + +ERROR 09-23 20:17:58 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERROR 09-23 20:17:58 [stage_runtime.py:265] File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\backends\llamacpp.py", line 177, in __init__ + +ERROR 09-23 20:17:58 [stage_runtime.py:265] raise RuntimeError("REFUSE_DEVICE_PLACEMENT: llama.cpp layer assignment differs from requested GPU") + +ERROR 09-23 20:17:58 [stage_runtime.py:265] RuntimeError: REFUSE_DEVICE_PLACEMENT: llama.cpp layer assignment differs from requested GPU +{ + "scope": "complete Spark GGUF text requests through Omni StageRuntime/StagePool and external llama.cpp backend", + "model_file": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "device": "Vulkan1", + "memory_budget": { + "capacities": { + "host_ram": 12884901888 + }, + "demands": { + "host_ram": 4294967296 + } + }, + "warmup_count": 0, + "measured_count": 1, + "status": "failed", + "error": "RuntimeError: REFUSE_DEVICE_PLACEMENT: llama.cpp layer assignment differs from requested GPU", + "ledger_after_shutdown": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } +} +Traceback (most recent call last): + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\benchmarks\edge_harness\experiments\probe_omni_llamacpp_spark.py", line 190, in + asyncio.run(main()) + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\runners.py", line 195, in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\zhout\AppData\Local\Programs\Python\Python312\Lib\asyncio\base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\benchmarks\edge_harness\experiments\probe_omni_llamacpp_spark.py", line 112, in main + runtime.initialize() + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 278, in initialize + raise exc + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\stage_runtime.py", line 817, in _initialize_replica + return create_graph_client( + ^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\backends\__init__.py", line 15, in create_graph_client + return LlamaCppTextStageClient(metadata, backend, ledger, reservation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\backends\llamacpp.py", line 177, in __init__ + raise RuntimeError("REFUSE_DEVICE_PLACEMENT: llama.cpp layer assignment differs from requested GPU") +RuntimeError: REFUSE_DEVICE_PLACEMENT: llama.cpp layer assignment differs from requested GPU diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_wrong_device_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_wrong_device_20260923.json new file mode 100644 index 000000000..75349871b --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_wrong_device_20260923.json @@ -0,0 +1,27 @@ +{ + "scope": "complete Spark GGUF text requests through Omni StageRuntime/StagePool and external llama.cpp backend", + "model_file": "C:\\Users\\zhout\\w2\\models\\Spark-X2.5-1.7B-Q4_K_M.gguf", + "device": "Vulkan1", + "memory_budget": { + "capacities": { + "host_ram": 12884901888 + }, + "demands": { + "host_ram": 4294967296 + } + }, + "warmup_count": 0, + "measured_count": 1, + "status": "failed", + "error": "RuntimeError: REFUSE_DEVICE_PLACEMENT: llama.cpp layer assignment differs from requested GPU", + "ledger_after_shutdown": { + "capacities": { + "host_ram": 12884901888 + }, + "reserved": { + "host_ram": 0 + }, + "owners": [], + "quarantined": [] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_wrong_device_20260923.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_wrong_device_20260923.log new file mode 100644 index 000000000..774dd5b92 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/omni_llamacpp_wrong_device_20260923.log @@ -0,0 +1,191 @@ +0.00.001.374 I srv llama_server: initializing ... +0.00.550.693 I cmn common_param: common_params_print_info: build 11124 (1a679828f) with Clang 20.1.8 for Windows x86_64 +0.00.550.704 I cmn common_param: common_params_print_info: verbosity = 4 (adjust with the `-lv N` CLI arg) +0.00.550.706 I cmn common_param: device_info: +0.00.555.940 I cmn common_param: - Vulkan0 : NVIDIA GeForce RTX 5090 Laptop GPU (24137 MiB, 23369 MiB free) +0.00.560.718 I cmn common_param: - Vulkan1 : AMD Radeon(TM) 890M Graphics (48505 MiB, 46080 MiB free) +0.00.560.736 I cmn common_param: - CPU : AMD Ryzen AI 9 HX 370 w/ Radeon 890M (64633 MiB, 27895 MiB free) +0.00.560.825 I cmn common_param: system_info: n_threads = 12 (n_threads_batch = 12) / 24 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.560.928 I srv init_listene: running without SSL +0.00.561.003 I srv init_listene: The UI is disabled +0.00.561.004 I srv init_listene: Use --ui/--no-ui (or deprecated --webui/--no-webui) to enable/disable +0.00.561.010 I srv init: using 23 threads for HTTP server +0.00.561.104 W srv llama_server: security: no API key is set and CORS allows all origins (see https://github.com/ggml-org/llama.cpp/pull/25655) +0.00.565.816 I srv load_model: loading model 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' +0.00.565.828 I srv load_model: local path 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' +0.00.605.525 I llama_model_loader: loaded meta data with 37 key-value pairs and 226 tensors from C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf (version GGUF V3 (latest)) +0.00.605.544 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +0.00.605.551 I llama_model_loader: - kv 0: general.architecture str = spark2_5 +0.00.605.551 I llama_model_loader: - kv 1: general.type str = model +0.00.605.555 I llama_model_loader: - kv 2: general.sampling.top_k i32 = 0 +0.00.605.563 I llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 +0.00.605.565 I llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 +0.00.605.566 I llama_model_loader: - kv 5: general.name str = Hf_Format +0.00.605.566 I llama_model_loader: - kv 6: general.size_label str = 1.7B +0.00.605.569 I llama_model_loader: - kv 7: spark2_5.block_count u32 = 28 +0.00.605.569 I llama_model_loader: - kv 8: spark2_5.context_length u32 = 1048576 +0.00.605.570 I llama_model_loader: - kv 9: spark2_5.embedding_length u32 = 2048 +0.00.605.570 I llama_model_loader: - kv 10: spark2_5.feed_forward_length u32 = 6656 +0.00.605.571 I llama_model_loader: - kv 11: spark2_5.attention.head_count u32 = 8 +0.00.605.571 I llama_model_loader: - kv 12: spark2_5.attention.head_count_kv u32 = 2 +0.00.605.573 I llama_model_loader: - kv 13: spark2_5.rope.freq_base f32 = 5000000.000000 +0.00.605.574 I llama_model_loader: - kv 14: spark2_5.rope.freq_base_swa f32 = 10000.000000 +0.00.605.576 I llama_model_loader: - kv 15: spark2_5.attention.layer_norm_rms_epsilon f32 = 0.000001 +0.00.605.577 I llama_model_loader: - kv 16: spark2_5.attention.key_length u32 = 256 +0.00.605.577 I llama_model_loader: - kv 17: spark2_5.attention.value_length u32 = 256 +0.00.605.578 I llama_model_loader: - kv 18: spark2_5.vocab_size u32 = 131072 +0.00.605.578 I llama_model_loader: - kv 19: spark2_5.attention.sliding_window u32 = 512 +0.00.605.602 I llama_model_loader: - kv 20: spark2_5.attention.sliding_window_pattern arr[bool,28] = [true, true, true, false, true, true,... +0.00.605.602 I llama_model_loader: - kv 21: spark2_5.rope.dimension_count u32 = 64 +0.00.605.603 I llama_model_loader: - kv 22: spark2_5.rope.dimension_count_swa u32 = 256 +0.00.605.603 I llama_model_loader: - kv 23: tokenizer.ggml.model str = gpt2 +0.00.605.604 I llama_model_loader: - kv 24: tokenizer.ggml.pre str = spark2_5 +0.00.631.421 I llama_model_loader: - kv 25: tokenizer.ggml.tokens arr[str,131072] = ["<｜start▁of▁sentence｜>", "<... +0.00.638.586 I llama_model_loader: - kv 26: tokenizer.ggml.token_type arr[i32,131072] = [3, 3, 3, 4, 4, 3, 3, 3, 4, 4, 4, 3, ... +0.00.662.312 I llama_model_loader: - kv 27: tokenizer.ggml.merges arr[str,130716] = ["Ġ Ġ", "Ġ t", "i n", "Ġ a", "Ġ... +0.00.662.329 I llama_model_loader: - kv 28: tokenizer.ggml.bos_token_id u32 = 0 +0.00.662.330 I llama_model_loader: - kv 29: tokenizer.ggml.eos_token_id u32 = 1 +0.00.662.333 I llama_model_loader: - kv 30: tokenizer.ggml.unknown_token_id u32 = 5 +0.00.662.333 I llama_model_loader: - kv 31: tokenizer.ggml.padding_token_id u32 = 2 +0.00.662.335 I llama_model_loader: - kv 32: tokenizer.ggml.add_bos_token bool = false +0.00.662.336 I llama_model_loader: - kv 33: tokenizer.ggml.add_eos_token bool = false +0.00.662.344 I llama_model_loader: - kv 34: tokenizer.chat_template str = {%- if not messages %}\n {{- raise_... +0.00.662.345 I llama_model_loader: - kv 35: general.quantization_version u32 = 2 +0.00.662.346 I llama_model_loader: - kv 36: general.file_type u32 = 15 +0.00.662.356 I llama_model_loader: - type f32: 57 tensors +0.00.662.357 I llama_model_loader: - type q4_K: 140 tensors +0.00.662.358 I llama_model_loader: - type q6_K: 29 tensors +0.00.662.364 I print_info: file format = GGUF V3 (latest) +0.00.662.365 I print_info: file type = Q4_K - Medium +0.00.662.371 I print_info: file size = 1.03 GiB (5.16 BPW) +0.00.667.972 I llama_prepare_model_devices: using device Vulkan1 (AMD Radeon(TM) 890M Graphics) (unknown id) - 46080 MiB free +0.00.815.731 I load: 86 unused tokens +0.00.853.251 I load: printing all EOG tokens: +0.00.853.261 I load: - 1 ('<｜end▁of▁sentence｜>') +0.00.853.450 I load: special tokens cache size = 113 +0.00.873.316 I load: token to piece cache size = 0.7975 MB +0.00.873.360 I print_info: arch = spark2_5 +0.00.873.361 I print_info: vocab_only = 0 +0.00.873.362 I print_info: no_alloc = 0 +0.00.873.362 I print_info: n_ctx_train = 1048576 +0.00.873.363 I print_info: n_embd_inp = 2048 +0.00.873.364 I print_info: n_embd = 2048 +0.00.873.364 I print_info: n_embd_out = 2048 +0.00.873.364 I print_info: n_layer = 28 +0.00.873.365 I print_info: n_layer_all = 28 +0.00.873.387 I print_info: n_head = 8 +0.00.873.389 I print_info: n_head_kv = 2 +0.00.873.389 I print_info: n_rot = 64 +0.00.873.390 I print_info: n_swa = 512 +0.00.873.390 I print_info: is_swa_any = 1 +0.00.873.391 I print_info: non_causal_type = 0 +0.00.873.391 I print_info: n_embd_head_k = 256 +0.00.873.392 I print_info: n_embd_head_v = 256 +0.00.873.393 I print_info: n_gqa = 4 +0.00.873.395 I print_info: n_embd_k_gqa = 512 +0.00.873.397 I print_info: n_embd_v_gqa = 512 +0.00.873.399 I print_info: f_norm_eps = 0.0e+00 +0.00.873.401 I print_info: f_norm_rms_eps = 1.0e-06 +0.00.873.401 I print_info: f_clamp_kqv = 0.0e+00 +0.00.873.402 I print_info: f_max_alibi_bias = 0.0e+00 +0.00.873.402 I print_info: f_logit_scale = 0.0e+00 +0.00.873.403 I print_info: f_attn_scale = 0.0e+00 +0.00.873.403 I print_info: f_attn_value_scale = 0.0000 +0.00.873.405 I print_info: n_ff = 6656 +0.00.873.406 I print_info: n_expert = 0 +0.00.873.406 I print_info: n_expert_used = 0 +0.00.873.409 I print_info: n_expert_groups = 0 +0.00.873.409 I print_info: n_group_used = 0 +0.00.873.410 I print_info: causal attn = 1 +0.00.873.410 I print_info: pooling type = -1 +0.00.873.411 I print_info: rope type = 2 +0.00.873.411 I print_info: rope scaling = linear +0.00.873.412 I print_info: freq_base_train = 5000000.0 +0.00.873.414 I print_info: freq_scale_train = 1 +0.00.873.415 I print_info: freq_base_swa = 10000.0 +0.00.873.416 I print_info: freq_scale_swa = 1 +0.00.873.416 I print_info: n_embd_head_k_swa = 256 +0.00.873.417 I print_info: n_embd_head_v_swa = 256 +0.00.873.417 I print_info: n_rot_swa = 256 +0.00.873.418 I print_info: n_ctx_orig_yarn = 1048576 +0.00.873.418 I print_info: rope_yarn_log_mul = 0.0000 +0.00.873.419 I print_info: rope_finetuned = unknown +0.00.873.420 I print_info: model type = 1.7B +0.00.873.421 I print_info: model params = 1.71 B +0.00.873.422 I print_info: general.name = Hf_Format +0.00.873.423 I print_info: vocab type = BPE +0.00.873.423 I print_info: n_vocab = 131072 +0.00.873.424 I print_info: n_merges = 130716 +0.00.873.425 I print_info: BOS token = 0 '<｜start▁of▁sentence｜>' +0.00.873.426 I print_info: EOS token = 1 '<｜end▁of▁sentence｜>' +0.00.873.426 I print_info: EOT token = 1 '<｜end▁of▁sentence｜>' +0.00.873.427 I print_info: UNK token = 5 '' +0.00.873.428 I print_info: PAD token = 2 '<｜▁pad▁｜>' +0.00.873.428 I print_info: LF token = 198 'Ċ' +0.00.873.429 I print_info: FIM PRE token = 11 '<｜fim▁begin｜>' +0.00.873.429 I print_info: FIM SUF token = 12 '<｜fim▁hole｜>' +0.00.873.430 I print_info: FIM MID token = 13 '<｜fim▁end｜>' +0.00.873.431 I print_info: EOG token = 1 '<｜end▁of▁sentence｜>' +0.00.873.431 I print_info: max token length = 210 +0.00.883.326 I load_tensors: loading model tensors, this can take a while... (load_mode = none) +0.01.410.499 I load_tensors: offloading output layer to GPU +0.01.410.508 I load_tensors: offloading 27 repeating layers to GPU +0.01.410.509 I load_tensors: offloaded 29/29 layers to GPU +0.01.410.516 I load_tensors: Vulkan1 model buffer size = 1051.02 MiB +0.01.410.517 I load_tensors: Vulkan_Host model buffer size = 210.00 MiB +0.01.920.346 I cmn common_init_: added <｜end▁of▁sentence｜> logit bias = -inf +0.01.920.932 I llama_context: constructing llama_context +0.01.920.946 I llama_context: n_seq_max = 1 +0.01.920.947 I llama_context: n_ctx = 4096 +0.01.920.948 I llama_context: n_ctx_seq = 4096 +0.01.920.949 I llama_context: n_batch = 2048 +0.01.920.949 I llama_context: n_ubatch = 512 +0.01.920.950 I llama_context: causal_attn = 1 +0.01.920.951 I llama_context: flash_attn = auto +0.01.920.952 I llama_context: kv_unified = false +0.01.920.956 I llama_context: freq_base = 5000000.0 +0.01.920.960 I llama_context: freq_scale = 1 +0.01.920.960 I llama_context: n_rs_seq = 0 +0.01.920.961 I llama_context: n_outputs_max = 1 +0.01.920.962 I llama_context: n_outputs_max_per_seq = 1 +0.01.920.963 I llama_context: n_ctx_seq (4096) < n_ctx_train (1048576) -- the full capacity of the model will not be utilized +0.01.921.413 I llama_context: Vulkan_Host output buffer size = 0.50 MiB +0.01.921.432 I llama_kv_cache_iswa: creating non-SWA KV cache, size = 4096 cells +0.01.932.943 I llama_kv_cache: Vulkan1 KV buffer size = 56.00 MiB +0.01.933.740 I llama_kv_cache: size = 56.00 MiB ( 4096 cells, 7 layers, 1/1 seqs), K (f16): 28.00 MiB, V (f16): 28.00 MiB +0.01.933.763 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.01.933.764 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.01.933.776 I llama_kv_cache_iswa: creating SWA KV cache, size = 1024 cells +0.01.942.111 I llama_kv_cache: Vulkan1 KV buffer size = 42.00 MiB +0.01.942.751 I llama_kv_cache: size = 42.00 MiB ( 1024 cells, 21 layers, 1/1 seqs), K (f16): 21.00 MiB, V (f16): 21.00 MiB +0.01.942.770 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 +0.01.942.770 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 +0.01.942.787 I sched_reserve: reserving ... +0.01.944.993 I resolve_fused_ops: Flash Attention enabled +0.01.945.000 I resolve_fused_ops: resolving fused DeepSeek V4 HC support: +0.01.946.371 I resolve_fused_ops: fused DeepSeek V4 HC pre enabled +0.01.947.684 I resolve_fused_ops: fused DeepSeek V4 HC comb enabled +0.01.948.989 I resolve_fused_ops: fused DeepSeek V4 HC post enabled +0.01.965.078 I sched_reserve: Vulkan1 compute buffer size = 56.02 MiB +0.01.965.088 I sched_reserve: Vulkan_Host compute buffer size = 13.02 MiB +0.01.965.090 I sched_reserve: graph: nodes = 986, splits = 2, input objects = 4, input tensors = 9 +0.01.965.092 I sched_reserve: reserve took 22.29 ms, sched copies = 1 +0.01.965.220 I cmn init: llama threadpool init, n_threads = 12 +0.01.965.240 I cmn common_init_: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.02.078.262 I srv load_model: initializing, n_slots = 1, n_ctx_slot = 4096, kv_unified = 'false' +0.02.078.300 I spec common_specu: no implementations specified for speculative decoding +0.02.078.302 I slot load_model: id 0 | task -1 | new slot, n_ctx = 4096 +0.02.078.538 I srv load_model: prompt cache is disabled - use `--cache-ram N` to enable it +0.02.078.546 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.02.078.547 I srv load_model: context checkpoints enabled, max = 32, min spacing = 8192 +0.02.078.579 W srv init: --cache-idle-slots requires --cache-ram, disabling +0.02.097.535 I srv init: init: chat template, example_format: '<｜start▁of▁sentence｜><|System|> +you are a helpful assistant. + +You are a helpful assistant<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>Hello<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>Hi there<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>How are you?<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>' +0.02.105.862 I srv init: init: chat template, thinking = 0 +0.02.105.912 I srv init: preserve_reasoning kwarg: true +0.02.105.914 W srv init: chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve) +0.02.105.971 I srv llama_server: model loaded +0.02.105.977 I srv llama_server: listening on http://127.0.0.1:62273 +0.02.106.153 I srv update_slots: all slots are idle diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md index db0c06c61..f6791ef51 100644 --- a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md @@ -1,6 +1,6 @@ # Spark-X2.5-1.7B Q4_K_M on native Windows CPU and Radeon 890M -This is a **scoped, complete text-model path in standalone llama.cpp**, with real weights, prefill, autoregressive decode, KV state and answer delivery. It is **not** yet an Omni `StageRuntime` backend, and it says nothing about other Spark variants or the TTS/MiniCPM/VLA cells. The current [device × model matrix](../../../e2e_profiling_20260922/evidence/summary/README.md) distinguishes this backend result from Omni integration. +This is the **historical standalone** scoped complete text-model path in llama.cpp, with real weights, prefill, autoregressive decode, KV state and answer delivery. A subsequent [bounded Omni whole-session stage](../spark_omni_llamacpp/README.md) now executes the same GGUF on native Windows CPU and Radeon 890M. Neither record establishes other Spark variants or the TTS/MiniCPM/VLA cells. The current [device × model matrix](../../../e2e_profiling_20260922/evidence/summary/README.md) distinguishes the two measurements. The GGUF is [XHToken/Spark-X2.5-1.7B-GGUF](https://huggingface.co/XHToken/Spark-X2.5-1.7B-GGUF/tree/1f7fa33b1245c14730da39e125714ad3a327901b) revision `1f7fa33b1245c14730da39e125714ad3a327901b`, file `Spark-X2.5-1.7B-Q4_K_M.gguf` (1,107,457,856 bytes; SHA-256 `902bde2522394954ac17821b3e5fd0df02defbc6944f122253f2580acf0503f4`). It remains in local model storage, not Git. The Windows Vulkan binary is [llama.cpp b11124](https://github.com/ggml-org/llama.cpp/releases/tag/b11124), build commit `1a679828f`; the release ZIP SHA-256 is `ece009eff2a18884246b4aef785ce4efd9eb36f4fc38e3d6727af6fe53bd2c75`. See the [artifact manifest](artifact_manifest.json) and [Vulkan enumeration](vulkaninfo_summary.log). The host was Windows 11 build 26200 on Ryzen AI 9 HX 370 with Radeon 890M driver `25.20.18.06 (LLPC)`. An RTX 5090 Laptop GPU was present, so the Radeon command explicitly selected `-dev Vulkan1`; the [placement log](spark_igpu_q4km_placement_20260923.log) records all 29 logged layer assignments on Vulkan1, a 1,051 MiB Vulkan1 model buffer and a separate 210 MiB Vulkan_Host buffer. The CPU command used `-dev none -ngl 0`. Both used a 4,096-token context, one slot, temperature zero, reasoning off for server requests, and default f16 KV cache. @@ -28,4 +28,4 @@ $model = 'C:\Users\zhout\w2\models\Spark-X2.5-1.7B-Q4_K_M.gguf' Run [the profile script](../../../../experiments/profile_llamacpp_spark_text.py) in another terminal with `--model-file $model`, `--expected-model-sha256 902bde2522394954ac17821b3e5fd0df02defbc6944f122253f2580acf0503f4`, `--profile-case inventory_120`, `--cancel-check`, and an `--output` path. For CPU, launch the same binary with `-dev none -ngl 0` on another port and set `--endpoint` accordingly. The fixed artifact, context, precision, prompt and cache policy are required for comparison. -The next project gate is to wrap this **whole autoregressive session** as a bounded llama.cpp stage behind Omni's request/event, memory-admission, state, streaming and cancellation contracts, then compare it with the current vLLM CPU/CUDA paths. The two-answer quality check, one repeated input, absent load-peak/power/thermal data, and client-close-only cancellation check do not establish broader release qualification. +The subsequent [Omni stage record](../spark_omni_llamacpp/README.md) closes the bounded complete-request binding and explicit admission gates. Incremental streaming, post-cancel restart, wider quality, load-peak/power/thermal data and comparison with the current vLLM CPU/CUDA paths remain open. This historical two-answer quality check, one repeated input and client-close-only cancellation check do not establish broader release qualification. diff --git a/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md b/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md index ff7e32fc9..3d8b5082a 100644 --- a/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md +++ b/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md @@ -1,18 +1,18 @@ # E2E check and profiling -All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [WSL CPU/native Windows expansion](../../../e2e_expansion_20260923/README.md). The expansion adds separate 20-request serial MiniCPM-o text-to-speech profiles on WSL CPU and native Windows GPU, a Spark output-head component profile on the HX370 AMD NPU, complete Spark text-request profiles through standalone llama.cpp on native Windows CPU and Radeon 890M, a standalone Qwen3-TTS text-to-WAV profile on native Windows CPU, and a separate hybrid CPU+Radeon 890M Qwen3-TTS profile. Profiling completion does not imply full release qualification or Omni integration. +All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [WSL CPU/native Windows expansion](../../../e2e_expansion_20260923/README.md). The expansion adds separate 20-request serial MiniCPM-o text-to-speech profiles on WSL CPU and native Windows GPU, a Spark output-head component profile on the HX370 AMD NPU, complete Spark text-request profiles through standalone llama.cpp and a bounded Omni whole-session stage on native Windows CPU and Radeon 890M, a standalone Qwen3-TTS text-to-WAV profile on native Windows CPU, and a separate hybrid CPU+Radeon 890M Qwen3-TTS profile. Profiling completion does not imply full release qualification. ## Configuration × model -NOT E2E means no complete device/model workload was verified. “Scoped E2E” identifies only the named input/output path; “standalone” means the whole model ran outside Omni and still needs a StageRuntime binding. “Synthetic policy pass” does not establish a real robot-task result. See the per-cell reasons below. +NOT E2E means no complete device/model workload was verified. “Scoped E2E” identifies only the named input/output path; “standalone” means the whole model ran outside Omni. The Spark GGUF Omni stage uses a complete-request control path; it has no incremental token stream or restart-after-cancel contract yet. “Synthetic policy pass” does not establish a real robot-task result. See the per-cell reasons below. | Device | Spark-X2.5 | Qwen3-TTS 0.6B CustomVoice | Qwen3.8-27B | MiniCPM-o 4.5 | InternVLA-A1 | |---|---|---|---|---|---| | PC HX370: CPU / WSL | profile completed | scoped E2E: short stream | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass | -| PC HX370: CPU / native Windows | scoped E2E: standalone Spark text | scoped E2E: standalone text → WAV | NOT E2E | NOT E2E | synthetic policy pass | +| PC HX370: CPU / native Windows | scoped E2E: Omni Spark text | scoped E2E: standalone text → WAV | NOT E2E | NOT E2E | synthetic policy pass | | PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text/image/audio → speech | synthetic policy pass | | PC HX370 + RTX 5090 Laptop / Windows | profile completed | profile completed | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass | -| PC HX370 + Radeon 890M / Windows worker | scoped E2E: standalone Spark text | scoped E2E: Radeon talker/codec + CPU predictor | NOT E2E | NOT E2E | NOT E2E | +| PC HX370 + Radeon 890M / Windows worker | scoped E2E: Omni Spark text | scoped E2E: Radeon talker/codec + CPU predictor | NOT E2E | NOT E2E | NOT E2E | | PC HX370 + AMD NPU / Windows worker | NOT E2E (output-head component passed) | NOT E2E | NOT E2E | NOT E2E | NOT E2E | | PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E | | PC Snapdragon X Elite CRD / AI Hub | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E | @@ -30,7 +30,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i | PC HX370: CPU / WSL | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No compatible CPU artifact/deployment for this model was found in the checked model roots. Available Qwen 27B FP8/NVFP4 CUDA artifacts do not establish a CPU route. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | PC HX370: CPU / WSL | MiniCPM-o 4.5 | SCOPED_E2E | The verified checkpoint completed real three-stage text, one synthetic image and one synthetic audio input to text+speech on the vLLM 0.28 CPU backend in separate bounded plans. The 448×448 red square was described correctly with a nonzero 5.2 s WAV in one 37.70 s request; the 2 s/440 Hz tone was described as “A loud and high pitched beep” with a nonzero 2.16 s WAV in one 17.30 s request. CPU wording/token IDs differ from the WSL GPU runs, with no task-quality threshold established. A separate 1-warmup/20-measured serial **text** run had request wall p50 21.86 s/p95 22.97 s. WSL swap reached 5.75 GB during that text profile; manual snapshots in the multimodal runs showed heavy swap use but do not prove peak memory. Both multimodal runs force-killed stage 0 at shutdown and reported shared-memory cleanup warnings. | Validate real image/audio suites, speech intelligibility, video input, concurrency, interruption, admission/loading peak, clean shutdown and sustained thermal behavior before broader CPU qualification; [CPU evidence](../../../e2e_expansion_20260923/README.md). | | PC HX370: CPU / WSL | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded in CPU-only PyTorch and produced finite `[1, 50, 32]` actions from zero observations/noise in one 3.25 s forward. Compared with the same CUDA checkpoint/input, relative L2 error was 1.30% and cosine similarity 0.999917; no task-quality tolerance is established. | Validate real observations against reference actions, units/timestamps, repeated latency and memory before robot-task qualification; [CPU evidence](../../../e2e_expansion_20260923/README.md). | -| PC HX370: CPU / native Windows | Spark-X2.5 | SCOPED_E2E_STANDALONE | The real Spark-X2.5-1.7B Q4_K_M GGUF ran a complete text path in native Windows llama.cpp CPU (`-dev none -ngl 0`): `Paris` and the correct `1511` answer to a 2,463-token inventory prompt, one warmup plus 20 uncached serial inventory requests with p50/p95 6.65/6.86 s and 20/20 correct, and a post-stream-close recovery request. This does not repair the missing vLLM CPU attention operators or establish an Omni stage binding. One repeated input, no loading peak or sustained thermal/power gate. | Bind the whole llama.cpp session to Omni's state, request/event, memory and cancellation contracts; validate broader text quality and sustained use; [standalone backend evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). | +| PC HX370: CPU / native Windows | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The pinned Spark-X2.5-1.7B Q4_K_M GGUF ran in native Windows llama.cpp CPU (`-dev none -ngl 0`) as a whole-session Omni StageRuntime/StagePool backend. `Paris` and the correct `1511` inventory answer passed; one warmup plus 20 serial complete requests were 20/20 correct, nearest-rank wall p50/p95 6.93/7.05 s. The public AsyncOmni short request passed. The stage verifies artifact hashes and CPU placement (0/29 layers offloaded), reserves 4 GiB host RAM, emits a terminal text event and releases reservation after in-flight cancellation. The separate prior standalone profile was 6.65/6.86 s. No vLLM CPU attention repair, incremental stream, restart-after-cancel, loading peak or sustained power/thermal gate is claimed. | Validate broader text/token quality, streaming and cancellation recovery, memory peak and sustained load; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). | | PC HX370: CPU / native Windows | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_STANDALONE | The pinned 0.6B CustomVoice checkpoint ran wholly on CPU through Qwen's standalone PyTorch wrapper (BF16/SDPA) and produced nonzero 24 kHz WAVs for two English sentences. For one repeated short prompt, 1 warmup + 20 serial measured complete generation requests had nearest-rank p50/p95 13.94/14.68 s for 4.56 s of audio. Independent Whisper tiny.en ASR gave word error rates 0.20 and 0.00 on the two WAVs; this is only an intelligibility proxy. The native vLLM CPU extension still lacks the required attention operators. No Omni binding, playable streaming, listening/quality threshold, loading peak or sustained power/thermal run was verified. | Bind the whole CPU TTS backend to Omni's stage and streaming/state contracts; validate playback, quality, cancellation, admission and sustained use; [standalone TTS evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md). | | PC HX370: CPU / native Windows | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The installed native Windows vLLM extension lacks cpu_attn_reshape_and_cache and cpu_attn_get_scheduler_metadata. The separate ORT CPU graph worker is not a whole-model text/TTS adapter; a qualified CPU build/binding is missing. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | PC HX370: CPU / native Windows | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | The exact checkpoint is verified in the local Linux model store, but the installed native Windows vLLM extension lacks required CPU attention operators; no native full-model run or memory admission exists. | Establish a native Windows CPU backend and model access, then run the full requested path; [weight evidence](../../../e2e_recovery_20260923/evidence/weights/minicpmo.json). | @@ -45,7 +45,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i | PC HX370 + RTX 5090 Laptop / Windows | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The Omni pipeline binding now exists and passed scoped text/image cases in WSL, but no native Windows Omni run has verified the NVFP4 checkpoint, memory admission or modality behavior. | Run the same checkpoint and explicit context on native Windows, then broaden quality and profiling; [WSL-only evidence](../../../e2e_recovery_20260923/README.md). | | PC HX370 + RTX 5090 Laptop / Windows | MiniCPM-o 4.5 | SCOPED_E2E | The verified checkpoint completed native Windows three-stage BF16 text, one synthetic image, and one synthetic audio input to text+speech in separate constrained plans with 5 GiB thinker host offload and explicit 2,048/512-token limits. Text and all 32 thinker token IDs matched WSL CUDA; a 5.2 s WAV was nonzero. The red-square image produced the same description and 27 thinker token IDs as WSL plus a nonzero 5.2 s WAV in 17.15 s. The 2 s/440 Hz tone produced the same whistling description and nine IDs as WSL plus a nonzero 2.16 s WAV in 8.26 s. The initial text run needed 900/600 s startup limits after a 300 s timeout; a separate 1-warmup/20-measured serial text run had request wall p50 18.38 s/p95 18.73 s, with sampled host available RAM as low as 0.78 GB and pagefile use up to 11.17 GB. These synthetic inputs do not establish general image/audio or speech quality. | Check real image/audio suites, video input, speech intelligibility/alignment, concurrency, interruption, loading-peak admission and sustained thermal/power behavior before broader qualification; [native evidence](../../../e2e_expansion_20260923/README.md). | | PC HX370 + RTX 5090 Laptop / Windows | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded from a UNC path and produced finite `[1, 50, 32]` actions on synthetic zero observations/noise. Policy/output CUDA placement was checked; one native forward took 1.12 s, with 6.68 GB peak allocated. Relative L2 difference from WSL CUDA was 0.83% (no task tolerance). | Validate real observations/reference actions, units/timestamps, memory and repeated latency before task qualification; [native evidence](../../../e2e_expansion_20260923/README.md). | -| PC HX370 + Radeon 890M / Windows worker | Spark-X2.5 | SCOPED_E2E_STANDALONE | The same real Spark-X2.5-1.7B Q4_K_M GGUF completed text prefill/decode via llama.cpp Vulkan1 on Radeon 890M; the placement log assigns Spark layers to Vulkan1, not the installed RTX 5090. It returned `Paris` and `1511` on the named short/2,463-token inventory inputs. One warmup plus 20 uncached serial inventory requests were 20/20 correct, p50/p95 wall 2.18/2.20 s. Closing a stream after its first content delta was followed by a correct new request. This is outside Omni and has only a post-profile process-memory snapshot, no loading peak, device-memory attribution or sustained power/thermal check. | Add a bounded llama.cpp whole-model stage to Omni, then verify placement, state, memory admission, token quality, cancellation and full-request benefit against CPU; [standalone backend evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). | +| PC HX370 + Radeon 890M / Windows worker | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The same pinned Spark-X2.5-1.7B Q4_K_M GGUF completed text prefill/decode through a bounded Omni whole-session stage with llama.cpp Vulkan1. The log identifies Radeon 890M and 29/29 layers offloaded there, rather than RTX 5090; an intentionally wrong device name was refused. `Paris` and `1511` passed; one warmup plus 20 serial complete requests were 20/20 correct, nearest-rank wall p50/p95 2.10/2.14 s. A public AsyncOmni request passed; an oversized context was rejected with a later correct request; in-flight cancellation produced no stale output and released the ledger. The prior standalone run was 2.18/2.20 s. No streaming, restart-after-cancel, GPU-memory peak or sustained power/thermal check is claimed. | Qualify public configuration, broader text/token quality, streaming and cancellation recovery, shared-RAM/GPU peak and sustained use; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). | | PC HX370 + Radeon 890M / Windows worker | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_STANDALONE_HYBRID | The Q8_0 CustomVoice GGUF and F16 codec GGUF completed two named text-to-WAV requests in standalone CrispASR on native Windows with Vulkan0 placed on Radeon 890M for the talker/codec and the 432 MB code predictor explicitly pinned to CPU FP32. Whisper tiny.en transcribed both sentences exactly (WER 0 as an intelligibility proxy). One warmup plus 20 serial resident-server requests for one repeated prompt had nearest-rank client wall p50/p95 3.310/3.345 s for identical 2.56 s PCM (RTF 1.29/1.31). The default Vulkan configuration actually fell back to CPU; forcing all stages to Vulkan ran away to the 360-frame cap. A separate DirectML route failed waveform quality. No Omni binding, playable streaming, loading peak, broad quality or sustained power/thermal run was verified. | Bind the whole hybrid session with explicit CPU+iGPU placement/admission and cancellation; validate quality, real-time playback and full-chain benefit before broader qualification; [joint backend evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_radeon890m_joint/README.md), [failed DirectML route](../../../e2e_expansion_20260923/evidence/qwen_tts_radeon890m_directml/README.md). | | PC HX370 + Radeon 890M / Windows worker | Qwen3.8-27B | NOT_E2E_PROFILEABLE | FP32 448x448 vision tower: DirectML 85/113 nodes; normalized max error 9.095e-06 versus torch CPU. Historical component path, not full 27B or current graph-stage image-to-text integration. | Recover the hashed tower bundle and qualify its current Omni graph handoff with the language stage. | | PC HX370 + Radeon 890M / Windows worker | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No placement- and quality-verified whole-model graph bundle/stateful adapter for this model on this AMD route. Small backend graphs do not establish model support. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | diff --git a/docs/design/omni_edge_backends.md b/docs/design/omni_edge_backends.md index 7238a0094..cb08163d6 100644 --- a/docs/design/omni_edge_backends.md +++ b/docs/design/omni_edge_backends.md @@ -1,10 +1,14 @@ # Local edge backends in Omni The first backend is `external.graph.v1`: one complete, stateless graph operation -per request, with a fixed input bucket and copied host tensors. It uses the normal -`Omni` / `AsyncOmni` APIs, `PipelineConfig`, `StageRuntime`, `StagePool` and -orchestrator. Native vLLM stages retain their model execution, KV cache, sampling -and batching. No accelerator is mandatory. +per request, with a fixed input bucket and copied host tensors. A second backend, +`external.llamacpp.text.v1`, owns one pinned GGUF whole-model server process and +returns complete text requests. Both use the normal `Omni` / `AsyncOmni` APIs, +`PipelineConfig`, `StageRuntime`, `StagePool` and orchestrator. Native vLLM stages +retain their model execution, KV cache, sampling and batching. No accelerator is +mandatory. The llama.cpp backend currently uses the complete-request graph +control path for admission and acknowledgement; it is not an ONNX graph and does +not yet support incremental token events or restart after active cancellation. ## Boundaries @@ -51,7 +55,7 @@ memory-accounting and unavailable-device gaps remain documented repair items. ### Mobile and PC × model support matrix -This section preserves the 2026-09-22 audit. For the latest per-cell disposition and 2026-09-23 follow-up evidence, use the [current 60-cell E2E matrix](../../benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md). The HX370 AMD NPU now has a placement- and quality-verified **Spark 1.7B output-head component** through Omni, while complete Spark generation on that route remains unverified; see the [component record](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_amd_npu_output_head/README.md). Native Windows CPU and Radeon 890M also completed [scoped whole-model Spark text requests](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md) with the same Q4_K_M GGUF in standalone llama.cpp. An Omni stage binding, explicit admission and broader quality gates remain open. These results advance the M3 hardware/backend evidence without claiming a joint multi-accelerator plan. +This section preserves the 2026-09-22 audit. For the latest per-cell disposition and 2026-09-23 follow-up evidence, use the [current 60-cell E2E matrix](../../benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md). The HX370 AMD NPU now has a placement- and quality-verified **Spark 1.7B output-head component** through Omni, while complete Spark generation on that route remains unverified; see the [component record](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_amd_npu_output_head/README.md). Native Windows CPU and Radeon 890M completed [scoped whole-model Spark text requests](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md) with the same Q4_K_M GGUF in standalone llama.cpp, then [complete requests through a bounded Omni stage](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md). The Omni route verifies pinned artifacts, actual placement, one-slot memory admission, terminal events and cancellation drain. Incremental streaming, restart after cancellation, memory peaks and broader quality remain open. These results advance the M3 hardware/backend evidence without claiming a joint multi-accelerator plan. **All 60 named pairings were checked on 2026-09-22** through actual runs, raw-record review or artifact/runtime preflight. This is not 60 successful diff --git a/docs/design/omni_edge_e2e_roadmap.md b/docs/design/omni_edge_e2e_roadmap.md index 17d21d055..46a137e26 100644 --- a/docs/design/omni_edge_e2e_roadmap.md +++ b/docs/design/omni_edge_e2e_roadmap.md @@ -131,6 +131,8 @@ precision and workload. New concurrency, state-boundary or quality evidence is required before broadening that acceptance. Current batch-dependent greedy outputs need reference analysis; timing alone cannot identify their cause. +The 2026-09-23 [Spark GGUF Omni follow-up](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md) adds a separate native Windows CPU/Radeon 890M whole-session option. Its pinned 1.7B Q4_K_M artifact passed two named complete text requests and 20 serial inventory requests per device through `StageRuntime`/`StagePool`; public `AsyncOmni` short requests passed on both devices. It verifies CPU or Vulkan1 placement, explicit 4 GiB host reservation, terminal event/acknowledgement and cancellation drain. This does not change the vLLM M0 acceptance or establish M1 mobile generation. Its next gates are incremental output, post-cancel restart, broader token/quality parity, loading/device-memory peaks, concurrency and sustained power/thermal behavior. + VLA qualification here ends at action data. Physical robot control, collision avoidance and actuator safety belong to the external controller. diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index d6b08a0b8..1f92d4a7d 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -112,7 +112,10 @@ from vllm_omni.model_executor.models.qwen3_omni.pipeline import ( resolve_qwen3_omni_pipeline, ) from vllm_omni.model_executor.models.qwen3_tts.pipeline import QWEN3_TTS_PIPELINE, QWEN3_TTS_TALKER_ONLY_PIPELINE -from vllm_omni.model_executor.models.spark2_5.pipeline import SPARK2_5_PIPELINE +from vllm_omni.model_executor.models.spark2_5.pipeline import ( + SPARK2_5_GGUF_TEXT_PIPELINE, + SPARK2_5_PIPELINE, +) from vllm_omni.model_executor.models.step_audio2.pipeline import ( STEP_AUDIO2_ASR_PIPELINE, STEP_AUDIO2_PIPELINE, @@ -147,6 +150,7 @@ OMNI_PIPELINES: dict[str, PipelineConfig | PipelineResolverFunc] = { "llama": LLAMA_PIPELINE, "qwen3_5": QWEN3_5_PIPELINE, "spark2_5": SPARK2_5_PIPELINE, + "spark2_5_gguf_text": SPARK2_5_GGUF_TEXT_PIPELINE, "step_audio_2": STEP_AUDIO2_PIPELINE, "step_audio_2_asr": STEP_AUDIO2_ASR_PIPELINE, "covo_audio": COVO_AUDIO_PIPELINE, diff --git a/vllm_omni/engine/backends/__init__.py b/vllm_omni/engine/backends/__init__.py index e183ad924..fb69290b9 100644 --- a/vllm_omni/engine/backends/__init__.py +++ b/vllm_omni/engine/backends/__init__.py @@ -3,8 +3,14 @@ def create_graph_client(metadata, config, ledger, reservation): - from .graph import GraphStageClient + backend = config.get("backend", {}) + name = backend.get("name") + if name == "external.graph.v1": + from .graph import GraphStageClient - if config.get("backend", {}).get("name") != "external.graph.v1": - raise ValueError("graph stage requires backend.name=external.graph.v1") - return GraphStageClient(metadata, config["backend"], ledger, reservation) + return GraphStageClient(metadata, backend, ledger, reservation) + if name == "external.llamacpp.text.v1": + from .llamacpp import LlamaCppTextStageClient + + return LlamaCppTextStageClient(metadata, backend, ledger, reservation) + raise ValueError(f"unsupported complete-request stage backend: {name!r}") diff --git a/vllm_omni/engine/backends/llamacpp.py b/vllm_omni/engine/backends/llamacpp.py new file mode 100644 index 000000000..3bb0f410f --- /dev/null +++ b/vllm_omni/engine/backends/llamacpp.py @@ -0,0 +1,378 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Bounded, complete-request llama.cpp text stage owned by StageRuntime. + +The llama-server subprocess owns its weights and KV. Omni owns the admission +reservation, request identity, output acknowledgement and process lifetime. +This v1 backend deliberately has one slot and no incremental output contract. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import hashlib +import json +import os +import re +import socket +import subprocess +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path +from typing import Any + +from omni_stage_contracts import StageEvent, StageRequest +from vllm.outputs import CompletionOutput + +from vllm_omni.engine.resource_ledger import ResourceUnavailable +from vllm_omni.engine.stage_client import StageClientBase +from vllm_omni.outputs import OmniRequestOutput + +_LAYER_ASSIGNMENT = re.compile(r"load_tensors: layer\s+\d+ assigned to device (\S+)") +_OFFLOADED_LAYERS = re.compile(r"load_tensors: offloaded (\d+)/(\d+) layers to GPU") +_MODEL_BUFFER = re.compile(r"load_tensors:\s+(Vulkan\d+) model buffer size") + + +def _digest(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(8 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def _free_local_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def _json_request(url: str, body: dict | None, *, timeout: float, limit: int) -> dict: + data = None if body is None else json.dumps(body).encode("utf-8") + request = urllib.request.Request( + url, + data=data, + headers={"Content-Type": "application/json"} if data is not None else {}, + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = response.read(limit + 1) + if len(payload) > limit: + raise ResourceUnavailable("llama.cpp response exceeds admitted I/O bound") + value = json.loads(payload) + if not isinstance(value, dict): + raise ValueError("llama.cpp response must be a JSON object") + return value + + +class LlamaCppTextStageClient(StageClientBase): + def __init__(self, metadata, config: dict, ledger, reservation) -> None: + for name, value in vars(metadata).items(): + setattr(self, name, value) + # The complete-request scheduler is reused; model execution is not an ONNX graph. + self.stage_type = "graph" + self._ledger, self._reservation = ledger, reservation + self._generation = uuid.uuid4().hex + self._config = dict(config) + self._proc: subprocess.Popen | None = None + self._log_stream = None + self._closed = False + self._active: str | None = None + self._task: asyncio.Task | None = None + self._output: OmniRequestOutput | None = None + self._epoch = 0 + self._max_io_bytes = int(config.get("max_io_bytes", 1 << 20)) + self._max_new_tokens = int(config.get("max_new_tokens", 96)) + self._context_tokens = int(config.get("context_tokens", 4096)) + self._request_timeout_s = float(config.get("request_timeout_s", 120)) + self._placement = str(config["device"]) + self._memory_pool = str(config.get("memory_pool", "host_ram")) + self._port = int(config.get("port") or _free_local_port()) + if ( + self._max_io_bytes <= 0 + or self._max_new_tokens <= 0 + or self._context_tokens <= self._max_new_tokens + or self._request_timeout_s <= 0 + or not 0 < self._port < 65536 + ): + raise ValueError("invalid llama.cpp request, context, timeout or port bound") + try: + if self._memory_pool not in reservation.demands: + raise ResourceUnavailable("llama.cpp memory pool absent from stage reservation") + if self._max_io_bytes > reservation.demands[self._memory_pool]: + raise ResourceUnavailable("llama.cpp I/O bound exceeds stage reservation") + self._model = Path(config["model_file"]).resolve(strict=True) + self._binary = Path(config["server_bin"]).resolve(strict=True) + self._log_path = Path(config["log_file"]).resolve() + self._expected_model_sha = str(config["model_sha256"]).lower() + self._expected_binary_sha = str(config["server_sha256"]).lower() + if _digest(self._model) != self._expected_model_sha or _digest(self._binary) != self._expected_binary_sha: + raise ValueError("llama.cpp executable or GGUF differs from the declared artifact hash") + overhead = int(config["memory_overhead_bytes"]) + if overhead <= 0 or self._model.stat().st_size + overhead > reservation.demands[self._memory_pool]: + raise ResourceUnavailable("GGUF plus declared KV/workspace/transfer/headroom exceeds reservation") + if self._placement != "cpu" and not re.fullmatch(r"Vulkan\d+", self._placement): + raise ValueError("llama.cpp device must be cpu or an explicit Vulkan index") + expected_device_name = config.get("expected_device_name") + if self._placement != "cpu" and not expected_device_name: + raise ValueError("Vulkan stage requires expected_device_name") + + self._log_path.parent.mkdir(parents=True, exist_ok=True) + self._log_stream = self._log_path.open("wb") + env = os.environ.copy() + if config.get("ggml_vk_visible_devices") is not None: + env["GGML_VK_VISIBLE_DEVICES"] = str(config["ggml_vk_visible_devices"]) + command = [ + str(self._binary), "-m", str(self._model), + "-dev", "none" if self._placement == "cpu" else self._placement, + "-ngl", "0" if self._placement == "cpu" else "99", + "-c", str(self._context_tokens), "-np", "1", + "--host", "127.0.0.1", "--port", str(self._port), + "--reasoning", "off", "--no-webui", "--fit", "off", + "--cache-ram", "0", "-lv", "4", + ] + flags = subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0 + self._proc = subprocess.Popen( + command, stdout=self._log_stream, stderr=subprocess.STDOUT, + env=env, creationflags=flags, + ) + base = f"http://127.0.0.1:{self._port}" + deadline = time.monotonic() + float(config.get("start_timeout_s", 120)) + while True: + if self._proc.poll() is not None: + raise RuntimeError(f"llama.cpp server exited during load; see {self._log_path}") + try: + health = _json_request(base + "/health", None, timeout=1, limit=4096) + if health.get("status") == "ok": + break + except (urllib.error.URLError, TimeoutError, OSError): + pass + if time.monotonic() >= deadline: + raise TimeoutError(f"llama.cpp server did not become ready; see {self._log_path}") + time.sleep(0.2) + props = _json_request(base + "/props", None, timeout=5, limit=64 << 10) + if Path(props.get("model_path", "")).resolve() != self._model or props.get("total_slots") != 1: + raise RuntimeError("llama.cpp did not load the pinned GGUF into one slot") + self._model_alias = props["model_alias"] + self._base_url = base + log_text = self._log_path.read_text(encoding="utf-8", errors="replace") + assignments = _LAYER_ASSIGNMENT.findall(log_text) + offloaded = _OFFLOADED_LAYERS.findall(log_text) + device_buffers = _MODEL_BUFFER.findall(log_text) + if self._placement == "cpu": + if ( + not offloaded or offloaded[-1][0] != "0" or device_buffers + or (assignments and any(name != "CPU" for name in assignments)) + ): + raise RuntimeError("CPU stage assigned model layers to an accelerator") + else: + expected_line = f"using device {self._placement} ({expected_device_name})" + fully_offloaded = bool(offloaded) and offloaded[-1][0] == offloaded[-1][1] + if ( + expected_line not in log_text or not fully_offloaded + or device_buffers != [self._placement] + or (assignments and any(name != self._placement for name in assignments)) + ): + raise RuntimeError("REFUSE_DEVICE_PLACEMENT: llama.cpp layer assignment differs from requested GPU") + self._loaded_rss_bytes = self._process_rss_bytes() + if self._loaded_rss_bytes > reservation.demands[self._memory_pool]: + raise ResourceUnavailable("loaded llama.cpp process RSS exceeds stage reservation") + self.execution_plan = { + "backend": "external.llamacpp.text.v1", + "stage_id": self.stage_id, + "worker_generation": self._generation, + "model_sha256": self._expected_model_sha, + "server_sha256": self._expected_binary_sha, + "model_alias": self._model_alias, + "requested_device": self._placement, + "expected_device_name": expected_device_name, + "layer_assignments": len(assignments), + "offloaded_layers": offloaded[-1], + "model_buffer_devices": device_buffers, + "worker_pid": self._proc.pid, + "loaded_rss_bytes": self._loaded_rss_bytes, + "reserved_bytes": dict(reservation.demands), + "memory_pool": self._memory_pool, + "memory_overhead_bytes": overhead, + "context_tokens": self._context_tokens, + "max_new_tokens": self._max_new_tokens, + "max_io_bytes": self._max_io_bytes, + "request_capacity": 1, + "stateful_session": "llama.cpp-owned; reset for each complete request", + "evidence": "B", + } + except BaseException: + self.shutdown() + raise + + def _process_rss_bytes(self) -> int: + import psutil + + return int(psutil.Process(self._proc.pid).memory_info().rss) + + async def add_request_async(self, request_id: str, prompt: Any, params: Any = None) -> None: + self.check_health() + if self._active is not None: + raise ResourceUnavailable("llama.cpp stage has an unacknowledged request; capacity is one") + if not isinstance(prompt, dict) or not isinstance(prompt.get("text"), str) or not prompt["text"]: + raise ValueError("llama.cpp prompt must contain nonempty text from its model adapter") + text = prompt["text"] + if len(text.encode("utf-8")) > self._max_io_bytes: + raise ResourceUnavailable("llama.cpp prompt exceeds admitted I/O bound") + max_tokens = int(prompt.get("max_tokens", self._max_new_tokens)) + if not 0 < max_tokens <= self._max_new_tokens: + raise ValueError("requested token limit exceeds admitted llama.cpp maximum") + temperature = float(prompt.get("temperature", 0)) + if temperature != 0: + raise ValueError("llama.cpp text v1 accepts deterministic temperature=0 only") + tokens = await asyncio.to_thread( + _json_request, + self._base_url + "/tokenize", + {"content": text}, + timeout=self._request_timeout_s, + limit=self._max_io_bytes, + ) + token_ids = tokens.get("tokens") + if not isinstance(token_ids, list) or not all(isinstance(item, int) for item in token_ids): + raise ValueError("llama.cpp returned invalid prompt tokenization") + # Leave a conservative allowance for the Spark chat template and stop markers. + if len(token_ids) + max_tokens + 64 > self._context_tokens: + raise ResourceUnavailable("Spark prompt plus generation bound exceeds context; refusing truncation") + self._epoch += 1 + epoch = self._epoch + request = StageRequest(request_id, self.stage_id, epoch, self._generation) + self._active = request_id + self._task = asyncio.create_task( + self._run(request, text, max_tokens), name=f"llamacpp-{self.stage_id}-{request_id}" + ) + + async def _run(self, request: StageRequest, text: str, max_tokens: int) -> None: + try: + started = time.perf_counter() + result = await asyncio.to_thread( + _json_request, + self._base_url + "/v1/chat/completions", + { + "model": self._model_alias, + "messages": [{"role": "user", "content": text}], + "temperature": 0, + "max_tokens": max_tokens, + "stream": False, + "cache_prompt": False, + }, + timeout=self._request_timeout_s, + limit=self._max_io_bytes, + ) + wall_s = time.perf_counter() - started + choice = result["choices"][0] + content = choice["message"]["content"] + finish_reason = choice["finish_reason"] + if not isinstance(content, str) or finish_reason != "stop": + raise RuntimeError(f"llama.cpp response incomplete: finish_reason={finish_reason!r}") + event = StageEvent( + request.request_id, self.stage_id, request.epoch, 1, "text", + self._generation, terminal=True, + ) + output = OmniRequestOutput( + request_id=request.request_id, + prompt=text, + stage_id=self.stage_id, + final_output_type="text", + outputs=[CompletionOutput(0, content, [], None, None, finish_reason=finish_reason)], + _custom_output={"stage_event": dataclasses.asdict(event)}, + metrics={"llamacpp_wall_s": wall_s, "usage": result.get("usage")}, + ) + except asyncio.CancelledError: + raise + except Exception as exc: + output = OmniRequestOutput.from_error(request.request_id, str(exc)) + output.stage_id = self.stage_id + if not self._closed and self._epoch == request.epoch: + loop = asyncio.get_running_loop() + output._stage_release = lambda: loop.call_soon_threadsafe( + self.acknowledge, request.request_id, request.epoch, request.worker_generation + ) + self._output = output + + def get_graph_output_nowait(self) -> OmniRequestOutput | None: + output, self._output = self._output, None + return output + + def acknowledge(self, request_id: str, epoch: int, generation: str) -> None: + if ( + self._active == request_id + and epoch == self._epoch + and generation == self._generation + and (self._task is None or self._task.done()) + ): + self._active = None + self._task = None + + async def abort_requests_async(self, request_ids: list[str]) -> None: + if self._active not in request_ids: + return + self._epoch += 1 + self._output = None + if self._task is not None and not self._task.done(): + drained = await asyncio.to_thread(self._terminate) + self._closed = True + done, _ = await asyncio.wait({self._task}, timeout=5) + if done: + self._ledger.release(self._reservation, drained=drained) + else: + self._ledger.release(self._reservation, drained=False) + self._task.add_done_callback( + lambda task: self._ledger.release( + self._reservation, drained=drained and not task.cancelled() + ) + ) + self._active = None + + def _terminate(self) -> bool: + proc = self._proc + if proc is None: + if self._log_stream is not None: + self._log_stream.close() + self._log_stream = None + return True + try: + if proc.poll() is None: + try: + proc.terminate() + except ProcessLookupError: + pass + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + except OSError: + return False + finally: + if self._log_stream is not None: + self._log_stream.close() + self._log_stream = None + return proc.poll() is not None + + def check_health(self) -> None: + if self._closed or self._proc is None or self._proc.poll() is not None: + from vllm.v1.engine.exceptions import EngineDeadError + + raise EngineDeadError() + + async def collective_rpc_async(self, method, timeout=None, args=(), kwargs=None): + raise NotImplementedError(f"llama.cpp backend does not implement collective RPC {method}") + + def shutdown(self) -> None: + self._closed = True + self._output = None + drained = self._terminate() + task = self._task + if task is not None and not task.done(): + self._ledger.release(self._reservation, drained=False) + task.add_done_callback( + lambda done: self._ledger.release(self._reservation, drained=drained and not done.cancelled()) + ) + else: + self._ledger.release(self._reservation, drained=drained) diff --git a/vllm_omni/model_executor/models/spark2_5/pipeline.py b/vllm_omni/model_executor/models/spark2_5/pipeline.py index ef52adc27..414e040a1 100644 --- a/vllm_omni/model_executor/models/spark2_5/pipeline.py +++ b/vllm_omni/model_executor/models/spark2_5/pipeline.py @@ -41,3 +41,20 @@ SPARK2_5_PIPELINE = PipelineConfig( ), ), ) + + +# A separate whole-session adapter for a pinned GGUF artifact. The complete +# request path reuses Omni's bounded graph-stage control plane, while llama.cpp +# owns the model weights, KV and sampling within the single backend process. +SPARK2_5_GGUF_TEXT_PIPELINE = PipelineConfig( + model_type="spark2_5_gguf_text", + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="text", + execution_type=StageExecutionType.GRAPH, + final_output=True, + final_output_type="text", + ), + ), +)