Add Omni MiniCPM-o GGUF whole-session CPU and Radeon paths
This commit is contained in:
tzhouam committed
1 parent
13cc723c71
commit
0a41206e1f
18 files changed
+2228
-7
No files matched your search
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a pinned audio+image→text+speech MiniCPM-o GGUF request through Omni.
|
||||
|
||||
The single C++ process owns the whole session. This probe records complete
|
||||
request latency; it does not claim incremental speech streaming.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
import wave
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for name in ("model-dir", "cli-bin", "reference-wav", "audio-wav", "image-jpeg",
|
||||
"artifact-report", "work-root", "log-root", "output-report", "output-wav"):
|
||||
parser.add_argument(f"--{name}", type=Path, required=True)
|
||||
parser.add_argument("--cli-sha256", required=True)
|
||||
parser.add_argument("--placement", choices=("cpu", "radeon-hybrid"), required=True)
|
||||
parser.add_argument("--reserve-gib", type=int, default=24)
|
||||
parser.add_argument("--capacity-gib", type=int, default=30)
|
||||
parser.add_argument("--abort-check", action="store_true")
|
||||
parser.add_argument("--expect-admission-refusal", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
from vllm_omni.config.stage_config import DeployConfig, StageDeployConfig, merge_pipeline_deploy
|
||||
from vllm_omni.engine.stage_runtime import StageRuntime
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO_4_5_GGUF_WHOLE_PIPELINE
|
||||
|
||||
source = json.loads(args.artifact_report.read_text(encoding="utf-8"))
|
||||
reference_hash = hashlib.sha256(args.reference_wav.read_bytes()).hexdigest()
|
||||
inputs = {"audio_wav": args.audio_wav.read_bytes(), "image_jpeg": args.image_jpeg.read_bytes()}
|
||||
backend = {
|
||||
"name": "external.minicpmo.gguf.v1",
|
||||
"placement": args.placement,
|
||||
"ggml_vk_visible_devices": "1" if args.placement == "radeon-hybrid" else None,
|
||||
"expected_gpu_name": "AMD Radeon(TM) 890M Graphics" if args.placement == "radeon-hybrid" else None,
|
||||
"model_dir": str(args.model_dir),
|
||||
"model_revision": source["model_revision"],
|
||||
"artifact_sha256": {name: data["sha256"] for name, data in source["artifacts"].items()},
|
||||
"cli_bin": str(args.cli_bin), "cli_sha256": args.cli_sha256,
|
||||
"reference_wav": str(args.reference_wav), "reference_sha256": reference_hash,
|
||||
"work_root": str(args.work_root), "log_root": str(args.log_root),
|
||||
"memory_overhead_bytes": 12 << 30,
|
||||
"context_tokens": 2048, "max_new_tokens": 96,
|
||||
"max_input_bytes": 2 << 20, "max_wav_bytes": 2 << 20,
|
||||
"max_text_bytes": 16 << 10, "max_work_bytes": 64 << 20,
|
||||
"max_input_audio_s": 10, "max_output_audio_s": 20,
|
||||
"request_timeout_s": 240, "t2w_wait_seconds": 180,
|
||||
}
|
||||
budget = {"capacities": {"host_ram": args.capacity_gib << 30},
|
||||
"demands": {"host_ram": args.reserve_gib << 30}}
|
||||
deploy = DeployConfig(async_chunk=False, stages=[StageDeployConfig(
|
||||
stage_id=0, backend=backend, resource_budget=budget,
|
||||
)])
|
||||
configs = [item.to_omegaconf() for item in merge_pipeline_deploy(
|
||||
MINICPMO_4_5_GGUF_WHOLE_PIPELINE, deploy,
|
||||
)]
|
||||
runtime = StageRuntime(configs, "local-minicpmo-gguf", "", stage_init_timeout=180,
|
||||
async_chunk=False)
|
||||
report = {"scope": "MiniCPM-o GGUF complete audio+image→text+speech through Omni StageRuntime/StagePool",
|
||||
"placement": args.placement, "model_revision": source["model_revision"],
|
||||
"artifact_sha256": backend["artifact_sha256"],
|
||||
"cli_sha256": args.cli_sha256, "reference_sha256": reference_hash,
|
||||
"input_sha256": {key: hashlib.sha256(value).hexdigest() for key, value in inputs.items()},
|
||||
"memory_budget": budget, "profile_count": 1, "warmup_count": 0}
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
if args.expect_admission_refusal:
|
||||
try:
|
||||
runtime.initialize()
|
||||
except Exception as exc:
|
||||
if "exceed reservation" not in str(exc):
|
||||
raise
|
||||
report["admission_refusal"] = f"{type(exc).__name__}: {exc}"
|
||||
report["status"] = "passed"
|
||||
return
|
||||
raise AssertionError("MiniCPM-o accepted an insufficient memory reservation")
|
||||
runtime.initialize()
|
||||
report["startup_s"] = time.perf_counter() - started
|
||||
pool = runtime.stage_pools[0]
|
||||
report["execution_plan_at_start"] = dict(pool.stage_client.execution_plan)
|
||||
state = SimpleNamespace(sampling_params_list=[None])
|
||||
|
||||
async def request_one(request_id: str):
|
||||
started = time.perf_counter()
|
||||
await pool.submit_initial(request_id, state, inputs)
|
||||
deadline = time.monotonic() + 250
|
||||
while True:
|
||||
output = pool.poll_graph_output(0)
|
||||
if output is not None:
|
||||
break
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError("MiniCPM-o Omni output timed out")
|
||||
await asyncio.sleep(0.01)
|
||||
try:
|
||||
if output.error:
|
||||
raise RuntimeError(output.error)
|
||||
completion = output.outputs[0]
|
||||
metadata = output.custom_output["audio_metadata"]
|
||||
pcm = (completion.multimodal_output["audio"].numpy() * 32768).astype("<i2").tobytes()
|
||||
assert completion.multimodal_output["sr"] == 24000
|
||||
assert len(pcm) // 2 == metadata["pcm_frames"]
|
||||
assert hashlib.sha256(pcm).hexdigest() == metadata["pcm_sha256"]
|
||||
assert "red" in completion.text.lower(), completion.text
|
||||
assert output.custom_output["stage_event"]["terminal"] is True
|
||||
return {"text": completion.text, "wall_s": time.perf_counter() - started,
|
||||
"audio_metadata": metadata, "stage_event": output.custom_output["stage_event"],
|
||||
"metrics": output.metrics, "pcm": pcm}
|
||||
finally:
|
||||
output.release_stage_buffers()
|
||||
await asyncio.sleep(0)
|
||||
|
||||
result = await request_one("minicpmo-1")
|
||||
pcm = result.pop("pcm")
|
||||
args.output_wav.parent.mkdir(parents=True, exist_ok=True)
|
||||
with wave.open(str(args.output_wav), "wb") as out:
|
||||
out.setnchannels(1)
|
||||
out.setsampwidth(2)
|
||||
out.setframerate(24000)
|
||||
out.writeframes(pcm)
|
||||
report["complete_request"] = result
|
||||
report["output_wav"] = str(args.output_wav)
|
||||
report["execution_plan_after_request"] = dict(pool.stage_client.execution_plan)
|
||||
if args.abort_check:
|
||||
await pool.submit_initial("minicpmo-abort", state, inputs)
|
||||
await asyncio.sleep(0.05)
|
||||
await pool.abort_requests(["minicpmo-abort"])
|
||||
await asyncio.sleep(0.1)
|
||||
report["abort_check"] = {
|
||||
"stale_output": pool.poll_graph_output(0) is not None,
|
||||
"ledger": runtime.resource_ledger.snapshot(),
|
||||
}
|
||||
assert not report["abort_check"]["stale_output"]
|
||||
report["status"] = "passed"
|
||||
except BaseException as exc:
|
||||
report["status"] = "failed"
|
||||
report["error"] = f"{type(exc).__name__}: {exc}"
|
||||
raise
|
||||
finally:
|
||||
runtime.shutdown()
|
||||
if runtime.resource_ledger is not None:
|
||||
report["ledger_after_shutdown"] = runtime.resource_ledger.snapshot()
|
||||
args.output_report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output_report.write_text(json.dumps(report, indent=2, default=str) + "\n", encoding="utf-8")
|
||||
print(json.dumps({key: value for key, value in report.items() if key != "artifact_sha256"},
|
||||
indent=2, default=str))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,9 +1,11 @@
|
||||
# 2026-09-23 WSL CPU and native Windows expansion
|
||||
|
||||
This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evidence/summary/README.md) with previously unrun WSL CPU and native Windows paths. The September 22 timing tables remain historical. MiniCPM-o now has separate 20-request serial text-to-speech timing runs on WSL CPU and native Windows GPU, plus one synthetic image and one synthetic audio input on each platform. A separate pinned GGUF route ran combined image+audio to text+speech requests on WSL CPU, native Windows CPU and native Windows Radeon 890M+CPU. The two Windows routes answered one spoken red-square question correctly with matching language token IDs; these C++ runs remain standalone, with broader input-grounded quality unqualified. Each InternVLA timing remains one observation. Spark has a 20-request Omni output-head component profile on the HX370 AMD NPU and separate 20-request **complete text** profiles through standalone llama.cpp on native Windows CPU and Radeon 890M. The latter now also run as bounded whole-session Omni stages, with public AsyncOmni requests checked separately. Qwen3-TTS has separate 20-request standalone and bounded Omni whole-session text-to-WAV profiles on native Windows CPU; the CPU+Radeon 890M hybrid route likewise has standalone and Omni profiles. None is a sustained or full-modality profile. The [checkpoint manifests](../e2e_recovery_20260923/README.md) pin the earlier real weights; the [Spark NPU component record](evidence/spark_amd_npu_output_head/README.md), [standalone CPU/Radeon whole-model record](evidence/spark_radeon890m_llamacpp/README.md), [Omni CPU/Radeon whole-session record](evidence/spark_omni_llamacpp/README.md), [standalone CPU Qwen TTS record](evidence/qwen_tts_native_cpu/README.md), [Omni CPU Qwen TTS record](evidence/qwen_tts_omni_cpu/README.md), [standalone joint CPU+Radeon TTS record](evidence/qwen_tts_radeon890m_joint/README.md), and [Omni joint TTS record](evidence/qwen_tts_omni_hybrid/README.md) pin their respective artifacts. No model, precision or context was silently substituted.
|
||||
This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evidence/summary/README.md) with previously unrun WSL CPU and native Windows paths. The September 22 timing tables remain historical. MiniCPM-o now has separate 20-request serial text-to-speech timing runs on WSL CPU and native Windows GPU, plus one synthetic image and one synthetic audio input on each platform. A separate pinned GGUF route ran combined image+audio to text+speech requests on WSL CPU, native Windows CPU and native Windows Radeon 890M+CPU. The two Windows routes answered one spoken red-square question correctly with matching language token IDs; those C++ routes now also pass a bounded complete request through [Omni whole-session stages](evidence/minicpmo_omni_cpp/README.md), with broader input-grounded quality unqualified. Each InternVLA timing remains one observation. Spark has a 20-request Omni output-head component profile on the HX370 AMD NPU and separate 20-request **complete text** profiles through standalone llama.cpp on native Windows CPU and Radeon 890M. The latter now also run as bounded whole-session Omni stages, with public AsyncOmni requests checked separately. Qwen3-TTS has separate 20-request standalone and bounded Omni whole-session text-to-WAV profiles on native Windows CPU; the CPU+Radeon 890M hybrid route likewise has standalone and Omni profiles. None is a sustained or full-modality profile. The [checkpoint manifests](../e2e_recovery_20260923/README.md) pin the earlier real weights; the [Spark NPU component record](evidence/spark_amd_npu_output_head/README.md), [standalone CPU/Radeon whole-model record](evidence/spark_radeon890m_llamacpp/README.md), [Omni CPU/Radeon whole-session record](evidence/spark_omni_llamacpp/README.md), [standalone CPU Qwen TTS record](evidence/qwen_tts_native_cpu/README.md), [Omni CPU Qwen TTS record](evidence/qwen_tts_omni_cpu/README.md), [standalone joint CPU+Radeon TTS record](evidence/qwen_tts_radeon890m_joint/README.md), and [Omni joint TTS record](evidence/qwen_tts_omni_hybrid/README.md) pin their respective artifacts. No model, precision or context was silently substituted.
|
||||
|
||||
| Device / model | Verified depth | Observed output and time | Remaining gate |
|
||||
|---|---|---|---|
|
||||
| Native Windows HX370 CPU / MiniCPM-o 4.5 Q4_K_M/F16 GGUF | Scoped complete-request Omni C++ whole-session graph stage (P for one synthetic combined audio+image input) | Correct red-square answer, 2.24 s 24 kHz WAV; one cold request 24.34 s, sampled RSS 13.11 GB. 24 GiB admission, 8 GiB refusal and in-flight cancellation with zero remaining reservation passed. | Real multimodal suites, playable streaming, repeated latency, loading peak, post-cancel restart and sustained power; [raw evidence](evidence/minicpmo_omni_cpp/README.md). |
|
||||
| Native Windows HX370 CPU+Radeon 890M / same MiniCPM-o GGUF | Scoped complete-request Omni hybrid graph stage (P for one synthetic combined audio+image input) | Correct red-square answer, 2.60 s 24 kHz WAV; one cold request 18.27 s, sampled RSS 15.57 GB. Log verifies Radeon language+vision and CPU TTS/Token2Wav; cancellation drained without stale output or reservation. | Real multimodal suites, playable streaming, repeated/concurrent latency, loading peak and sustained power; [raw evidence](evidence/minicpmo_omni_cpp/README.md). |
|
||||
| Native Windows HX370 CPU+Radeon 890M / Qwen3-TTS 0.6B CustomVoice Q8_0 GGUF plus F16 codec | Scoped complete standalone CrispASR text-to-WAV path with Vulkan talker/codec and CPU FP32 code predictor (P for two named sentences; outside Omni; **not** all-Radeon or NPU) | Two 24 kHz WAVs of 2.56 s and 3.20 s; independently pinned Whisper tiny.en ASR WER 0 on both. One warmup + 20 serial measured resident-server requests for the first prompt had client wall p50/p95 3.310/3.345 s, all PCM hashes identical; RTF 1.29/1.31. The default Vulkan setting fell back to CPU, while all-Vulkan talker+predictor hit a 360-frame runaway cap. | Explicit Omni hybrid stage binding, quality and playback gates, loading-peak admission, cancellation, broader prompts, concurrency and sustained power/thermal behavior; [joint raw evidence](evidence/qwen_tts_radeon890m_joint/README.md), [failed DirectML attempt](evidence/qwen_tts_radeon890m_directml/README.md). |
|
||||
| Native Windows HX370 CPU+Radeon 890M / same Qwen3-TTS 0.6B CustomVoice GGUFs | Scoped complete-request Omni CrispASR hybrid stage (P for two named sentences; **not** all-Radeon or NPU) | Both 24 kHz PCM outputs exactly match the earlier standalone hashes. One warmup + 20 serial Omni requests for 2.56 s of audio had 20/20 identical PCM, nearest-rank wall p50/p95 3.298/3.336 s (RTF 1.29/1.30). Public AsyncOmni audio passed; cancellation drained the worker and ledger. | Playable streaming, post-cancel restart, broad speech quality, loading/iGPU peak, concurrency and sustained real-time/thermal gates; [Omni raw evidence](evidence/qwen_tts_omni_hybrid/README.md). |
|
||||
| Native Windows HX370 CPU / Qwen3-TTS 0.6B CustomVoice, revision `85e237c12c027371202489a0ec509ded67b5e4b5` | Scoped complete standalone PyTorch text-to-WAV path (P for two named sentences; outside Omni) | Nonzero 24 kHz WAVs of 4.56 s and 3.44 s. For the first prompt, 1 warmup + 20 serial measured complete generations p50/p95 13.94/14.68 s; Whisper tiny.en ASR WER 0.20 on the first and 0.00 on the second sentence. | See the subsequent Omni CPU binding below; playable streaming, speech quality, memory loading peak and sustained behavior remain open; [standalone raw evidence](evidence/qwen_tts_native_cpu/README.md). |
|
||||
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
# MiniCPM-o GGUF through an Omni whole-session stage
|
||||
|
||||
These are two scoped, complete **audio+image → text+speech** requests through
|
||||
`StageRuntime`/`StagePool` and `external.minicpmo.gguf.v1`, on the native
|
||||
Windows 11 build 26200 HX370 laptop. They are not incremental audio streams or
|
||||
full vLLM model execution. The C++ worker owns the model session and speech
|
||||
state for each request; Omni verifies artifacts and placement, reserves shared
|
||||
RAM, controls the request and cancellation, and emits a terminal audio event.
|
||||
|
||||
The Q4_K_M language plus nine F16 GGUF companions are the exact ten files and
|
||||
revision `db25077c33951fe163b42986fba0132e279872a2` pinned in the
|
||||
[CPU](cpu_report.json) and [Radeon](radeon_report.json) reports. The external
|
||||
C++ source is `tc-mb/llama.cpp-omni` at
|
||||
`5202b7b2f4d11f50b9f996161e7a2f8b8571b890`, with the
|
||||
[recorded patch](llama_cpp_omni_bounded.patch) adding portable Windows
|
||||
completion and a 96-token generation bound. Rebuild the CPU and Vulkan CLI
|
||||
with the [recorded cross-build configuration](../minicpmo_cpp_radeon890m/README.md);
|
||||
these new binary SHA256 values are pinned in the JSON reports. Native Windows Python used the
|
||||
installed vLLM 0.29 environment with this Omni checkout. The input is the
|
||||
[same 16 kHz spoken question and 448×448 red-square JPEG](../minicpmo_cpp_radeon890m_prompted/README.md)
|
||||
used by the standalone parity run; the source speech was independently
|
||||
transcribed as “What color is the square in the image?”.
|
||||
|
||||
| Placement | Complete-request wall time | Output | Sampled worker RSS peak | Raw report |
|
||||
|---|---:|---|---:|---|
|
||||
| Native Windows CPU only | 24.34 s | “The square in the image is red.”; [2.24 s WAV](cpu_output.wav) | 13.11 GB | [JSON](cpu_report.json) |
|
||||
| Radeon 890M Vulkan0 language+vision, CPU TTS/Token2Wav/vocoder | 18.27 s | Same text; [2.60 s WAV](radeon_output.wav) | 15.57 GB | [JSON](radeon_report.json) |
|
||||
|
||||
Each has **one cold request, zero warmups, concurrency one**. The C++ CLI uses
|
||||
2,048 context tokens, `--max-new-tokens 96`, 180 s bounded Token2Wav wait,
|
||||
and a 240 s complete-request timeout. The stage reserves 24 GiB from a
|
||||
declared 30 GiB shared-RAM capacity, including 8.87 GB weights, 12 GiB
|
||||
headroom/state/workspace allowance, and 64 MiB bounded temporary output.
|
||||
It checks mono PCM16 16 kHz input, 448×448 JPEG, output text and complete
|
||||
24 kHz nonzero WAV chunks. Both logs show index-1 audio and vision prefill and
|
||||
`Audio generation completed.` The [CPU logs](logs_cpu/) show zero GPU layers,
|
||||
CPU vision and CPU Token2Wav. The [Radeon logs](logs_radeon/) show Radeon 890M
|
||||
selected by `GGML_VK_VISIBLE_DEVICES=1`, 37/37 language layers and vision on
|
||||
Vulkan0, 0/21 TTS layers on GPU, and CPU vocoder. Thus the Radeon route is
|
||||
CPU+iGPU joint execution, not all-GPU or NPU execution.
|
||||
|
||||
Each run emitted one terminal audio event, reported output/input hashes, then
|
||||
cancelled a new in-flight request: no stale output appeared, and the resource
|
||||
ledger returned to zero. An intentionally insufficient 8 GiB CPU reservation
|
||||
was [refused before worker launch](low_budget_report.json), with no remaining
|
||||
ledger owner. The worker logs, reports, generated WAVs and full C++ patch are
|
||||
retained here. The upstream optional WAV merger is not used; the stage verifies
|
||||
and assembles contiguous output chunks after the completion marker.
|
||||
|
||||
The standalone outputs for this fixture were independently transcribed with
|
||||
Whisper tiny.en at WER 0, an intelligibility proxy; these new Omni WAVs were
|
||||
checked for nonzero complete PCM but not independently ASR-scored. One case
|
||||
does not establish quality across real images or speech, loading peak,
|
||||
repeatability, playable streaming, cancellation restart, concurrent admission,
|
||||
or sustained power and thermals. No power condition was controlled.
|
||||
|
||||
Reproduce with
|
||||
[`probe_omni_minicpmo_cpp.py`](../../../../experiments/probe_omni_minicpmo_cpp.py)
|
||||
using `--placement cpu` or `--placement radeon-hybrid`, the paths and hashes in
|
||||
the JSON reports, the referenced model revision and the named audio/JPEG
|
||||
fixture. Add `--abort-check` for cancellation and
|
||||
`--reserve-gib 8 --expect-admission-refusal` for the budget refusal. The
|
||||
Windows Python process needs `PYTHONUTF8=1` and `PYTHONPATH` pointing to this
|
||||
Omni checkout.
|
||||
BIN
Binary file not shown.
+153
@@ -0,0 +1,153 @@
|
||||
{
|
||||
"scope": "MiniCPM-o GGUF complete audio+image\u2192text+speech through Omni StageRuntime/StagePool",
|
||||
"placement": "cpu",
|
||||
"model_revision": "db25077c33951fe163b42986fba0132e279872a2",
|
||||
"artifact_sha256": {
|
||||
"MiniCPM-o-4_5-Q4_K_M.gguf": "1237a97ee081b8abebc47aa7dad565701e8f5f904cdc92f6723ac4281bbc0932",
|
||||
"audio/MiniCPM-o-4_5-audio-F16.gguf": "d5b188ac7feaf98e17175c3f9bd14bf269301bfd187439fdaa3e3a494fc32ef7",
|
||||
"vision/MiniCPM-o-4_5-vision-F16.gguf": "1453678cc4e4fe18de241952962e234f265cb8dda780773526103ab8ba82f421",
|
||||
"tts/MiniCPM-o-4_5-tts-F16.gguf": "c7be3748a863dd6966ae7eed42600b7f41ca67affb03729ff245247f0e5ea088",
|
||||
"tts/MiniCPM-o-4_5-projector-F16.gguf": "4b1b5b377358a5e594a304ff6ea5d52df606a9ba7d886c4299d232f0c67dd1fd",
|
||||
"token2wav-gguf/encoder.gguf": "7f8d265da594eaf5e1de2db8f5f1867dbcb0bb75ef5878fadf2952347116f4d0",
|
||||
"token2wav-gguf/flow_matching.gguf": "eda6069f3edeb5dd3a87fbf2aedb2ddd1b46f3273926c4fcf09b24476a39cab8",
|
||||
"token2wav-gguf/flow_extra.gguf": "c67611aa7d02500fe395a7798bf0bfdfb55c74d37ba93934ca74d82b4e63f78d",
|
||||
"token2wav-gguf/hifigan2.gguf": "1b8b3bf5d8d3066aeee4324fdcdd41aefce170d0ee907645858de408d82835c2",
|
||||
"token2wav-gguf/prompt_cache.gguf": "81fe6f541ebe0b67db06a1e395df928da47991dc9637c2cf47d6c59d5b979f2c"
|
||||
},
|
||||
"cli_sha256": "29247187ddf3b13591cd14cc070e5e25c6bb7547296acc5e8a74e2945811d725",
|
||||
"reference_sha256": "cf4bb5cd9acbfbf67596d4a53310652b61ef4b7ede1d54eb03ef4882ca40223a",
|
||||
"input_sha256": {
|
||||
"audio_wav": "14ee147cbf2ce9afd9e32c8c8228d7cfe11985cd7e63456b2ad33871492aefcc",
|
||||
"image_jpeg": "22c7ccea466a3f551c512045e6316c7597408d4e0a20205502f55784d977aa8d"
|
||||
},
|
||||
"memory_budget": {
|
||||
"capacities": {
|
||||
"host_ram": 32212254720
|
||||
},
|
||||
"demands": {
|
||||
"host_ram": 25769803776
|
||||
}
|
||||
},
|
||||
"profile_count": 1,
|
||||
"warmup_count": 0,
|
||||
"startup_s": 6.932118400000036,
|
||||
"execution_plan_at_start": {
|
||||
"backend": "external.minicpmo.gguf.v1",
|
||||
"stage_id": 0,
|
||||
"worker_generation": "506f17f8d6044e468f18ff8a6bee5466",
|
||||
"model_revision": "db25077c33951fe163b42986fba0132e279872a2",
|
||||
"artifact_sha256": {
|
||||
"MiniCPM-o-4_5-Q4_K_M.gguf": "1237a97ee081b8abebc47aa7dad565701e8f5f904cdc92f6723ac4281bbc0932",
|
||||
"audio/MiniCPM-o-4_5-audio-F16.gguf": "d5b188ac7feaf98e17175c3f9bd14bf269301bfd187439fdaa3e3a494fc32ef7",
|
||||
"vision/MiniCPM-o-4_5-vision-F16.gguf": "1453678cc4e4fe18de241952962e234f265cb8dda780773526103ab8ba82f421",
|
||||
"tts/MiniCPM-o-4_5-tts-F16.gguf": "c7be3748a863dd6966ae7eed42600b7f41ca67affb03729ff245247f0e5ea088",
|
||||
"tts/MiniCPM-o-4_5-projector-F16.gguf": "4b1b5b377358a5e594a304ff6ea5d52df606a9ba7d886c4299d232f0c67dd1fd",
|
||||
"token2wav-gguf/encoder.gguf": "7f8d265da594eaf5e1de2db8f5f1867dbcb0bb75ef5878fadf2952347116f4d0",
|
||||
"token2wav-gguf/flow_matching.gguf": "eda6069f3edeb5dd3a87fbf2aedb2ddd1b46f3273926c4fcf09b24476a39cab8",
|
||||
"token2wav-gguf/flow_extra.gguf": "c67611aa7d02500fe395a7798bf0bfdfb55c74d37ba93934ca74d82b4e63f78d",
|
||||
"token2wav-gguf/hifigan2.gguf": "1b8b3bf5d8d3066aeee4324fdcdd41aefce170d0ee907645858de408d82835c2",
|
||||
"token2wav-gguf/prompt_cache.gguf": "81fe6f541ebe0b67db06a1e395df928da47991dc9637c2cf47d6c59d5b979f2c"
|
||||
},
|
||||
"cli_sha256": "29247187ddf3b13591cd14cc070e5e25c6bb7547296acc5e8a74e2945811d725",
|
||||
"requested_placement": "cpu",
|
||||
"ggml_vk_visible_devices": null,
|
||||
"shared_ram_reservation": {
|
||||
"host_ram": 25769803776
|
||||
},
|
||||
"weights_bytes": 8872297088,
|
||||
"memory_overhead_bytes": 12884901888,
|
||||
"context_tokens": 2048,
|
||||
"max_new_tokens": 96,
|
||||
"request_capacity": 1,
|
||||
"stateful_session": "C++-owned within one request; discarded after completion",
|
||||
"placement_evidence": "pending first request"
|
||||
},
|
||||
"complete_request": {
|
||||
"text": "The square in the image is red.",
|
||||
"wall_s": 24.338328399928287,
|
||||
"audio_metadata": {
|
||||
"sample_rate_hz": 24000,
|
||||
"channels": 1,
|
||||
"pcm_frames": 53760,
|
||||
"duration_s": 2.24,
|
||||
"pcm_sha256": "e4366dbc55177d11b2a8902c07685fcff2d3412abfaa3feb85b095672484f141"
|
||||
},
|
||||
"stage_event": {
|
||||
"request_id": "minicpmo-1",
|
||||
"stage_id": 0,
|
||||
"epoch": 1,
|
||||
"seq": 1,
|
||||
"kind": "audio",
|
||||
"worker_generation": "506f17f8d6044e468f18ff8a6bee5466",
|
||||
"buffers": [],
|
||||
"terminal": true,
|
||||
"error": null
|
||||
},
|
||||
"metrics": {
|
||||
"minicpmo_whole_request_wall_s": 24.305023799999617,
|
||||
"peak_sampled_rss_bytes": 13107023872,
|
||||
"worker_log": "C:\\Users\\zhout\\w2\\minicpmo_omni_logs_cpu\\minicpmo-1-478e965280734397918e5b3687e11006.log"
|
||||
}
|
||||
},
|
||||
"output_wav": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\results\\e2e_expansion_20260923\\evidence\\minicpmo_omni_cpp\\cpu_output.wav",
|
||||
"execution_plan_after_request": {
|
||||
"backend": "external.minicpmo.gguf.v1",
|
||||
"stage_id": 0,
|
||||
"worker_generation": "506f17f8d6044e468f18ff8a6bee5466",
|
||||
"model_revision": "db25077c33951fe163b42986fba0132e279872a2",
|
||||
"artifact_sha256": {
|
||||
"MiniCPM-o-4_5-Q4_K_M.gguf": "1237a97ee081b8abebc47aa7dad565701e8f5f904cdc92f6723ac4281bbc0932",
|
||||
"audio/MiniCPM-o-4_5-audio-F16.gguf": "d5b188ac7feaf98e17175c3f9bd14bf269301bfd187439fdaa3e3a494fc32ef7",
|
||||
"vision/MiniCPM-o-4_5-vision-F16.gguf": "1453678cc4e4fe18de241952962e234f265cb8dda780773526103ab8ba82f421",
|
||||
"tts/MiniCPM-o-4_5-tts-F16.gguf": "c7be3748a863dd6966ae7eed42600b7f41ca67affb03729ff245247f0e5ea088",
|
||||
"tts/MiniCPM-o-4_5-projector-F16.gguf": "4b1b5b377358a5e594a304ff6ea5d52df606a9ba7d886c4299d232f0c67dd1fd",
|
||||
"token2wav-gguf/encoder.gguf": "7f8d265da594eaf5e1de2db8f5f1867dbcb0bb75ef5878fadf2952347116f4d0",
|
||||
"token2wav-gguf/flow_matching.gguf": "eda6069f3edeb5dd3a87fbf2aedb2ddd1b46f3273926c4fcf09b24476a39cab8",
|
||||
"token2wav-gguf/flow_extra.gguf": "c67611aa7d02500fe395a7798bf0bfdfb55c74d37ba93934ca74d82b4e63f78d",
|
||||
"token2wav-gguf/hifigan2.gguf": "1b8b3bf5d8d3066aeee4324fdcdd41aefce170d0ee907645858de408d82835c2",
|
||||
"token2wav-gguf/prompt_cache.gguf": "81fe6f541ebe0b67db06a1e395df928da47991dc9637c2cf47d6c59d5b979f2c"
|
||||
},
|
||||
"cli_sha256": "29247187ddf3b13591cd14cc070e5e25c6bb7547296acc5e8a74e2945811d725",
|
||||
"requested_placement": "cpu",
|
||||
"ggml_vk_visible_devices": null,
|
||||
"shared_ram_reservation": {
|
||||
"host_ram": 25769803776
|
||||
},
|
||||
"weights_bytes": 8872297088,
|
||||
"memory_overhead_bytes": 12884901888,
|
||||
"context_tokens": 2048,
|
||||
"max_new_tokens": 96,
|
||||
"request_capacity": 1,
|
||||
"stateful_session": "C++-owned within one request; discarded after completion",
|
||||
"placement_evidence": {
|
||||
"observed": "cpu",
|
||||
"worker_pid": 66164,
|
||||
"peak_sampled_rss_bytes": 13107023872,
|
||||
"worker_log": "C:\\Users\\zhout\\w2\\minicpmo_omni_logs_cpu\\minicpmo-1-478e965280734397918e5b3687e11006.log"
|
||||
}
|
||||
},
|
||||
"abort_check": {
|
||||
"stale_output": false,
|
||||
"ledger": {
|
||||
"capacities": {
|
||||
"host_ram": 32212254720
|
||||
},
|
||||
"reserved": {
|
||||
"host_ram": 0
|
||||
},
|
||||
"owners": [],
|
||||
"quarantined": []
|
||||
}
|
||||
},
|
||||
"status": "passed",
|
||||
"ledger_after_shutdown": {
|
||||
"capacities": {
|
||||
"host_ram": 32212254720
|
||||
},
|
||||
"reserved": {
|
||||
"host_ram": 0
|
||||
},
|
||||
"owners": [],
|
||||
"quarantined": []
|
||||
}
|
||||
}
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
diff --git a/tools/omni/omni-cli.cpp b/tools/omni/omni-cli.cpp
|
||||
index 2e9623a..2b657a8 100644
|
||||
--- a/tools/omni/omni-cli.cpp
|
||||
+++ b/tools/omni/omni-cli.cpp
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
#include <iostream>
|
||||
#include <chrono>
|
||||
+#include <thread>
|
||||
#include <vector>
|
||||
#include <limits.h>
|
||||
#include <cinttypes>
|
||||
@@ -60,7 +61,10 @@ static void show_usage(const char * prog_name) {
|
||||
" --projector <path> Override projector model path\n"
|
||||
" --ref-audio <path> Reference audio for voice cloning (default: tools/omni/assets/default_ref_audio/default_ref_audio.wav)\n"
|
||||
" -c, --ctx-size <n> Context size (default: 4096)\n"
|
||||
+ " --max-new-tokens <n> Bound language generation to n tokens (default: context size)\n"
|
||||
" -ngl <n> Number of GPU layers (default: 99)\n"
|
||||
+ " --t2w-device <mode> Token2Wav device: cpu or gpu:<index> (default: gpu:0)\n"
|
||||
+ " --t2w-wait-seconds <n> Fail if async audio is incomplete after n seconds (default: 120)\n"
|
||||
" --no-tts Disable TTS output\n"
|
||||
" --omni Enable omni mode (audio + vision, media_type=2)\n"
|
||||
" --vision-backend <mode> Vision compute backend: 'metal' (default) or 'coreml' (ANE)\n"
|
||||
@@ -208,7 +212,10 @@ int main(int argc, char ** argv) {
|
||||
std::string vision_coreml_model_path; // CoreML model path (required when vision_backend=coreml)
|
||||
std::string ref_audio_path = "tools/omni/assets/default_ref_audio/default_ref_audio.wav";
|
||||
int n_ctx = 4096;
|
||||
+ int max_new_tokens = -1;
|
||||
int n_gpu_layers = 99; // GPU 层数，默认 99
|
||||
+ std::string token2wav_device = "gpu:0";
|
||||
+ int token2wav_wait_seconds = 120;
|
||||
int media_type = 1; // 1=audio only, 2=omni (audio+vision)
|
||||
bool use_tts = true;
|
||||
bool run_test = false;
|
||||
@@ -244,9 +251,30 @@ int main(int argc, char ** argv) {
|
||||
else if ((arg == "-c" || arg == "--ctx-size") && i + 1 < argc) {
|
||||
n_ctx = std::atoi(argv[++i]);
|
||||
}
|
||||
+ else if (arg == "--max-new-tokens" && i + 1 < argc) {
|
||||
+ max_new_tokens = std::atoi(argv[++i]);
|
||||
+ if (max_new_tokens < 1 || max_new_tokens >= n_ctx) {
|
||||
+ fprintf(stderr, "Error: --max-new-tokens must be 1..ctx-size-1\n");
|
||||
+ return 1;
|
||||
+ }
|
||||
+ }
|
||||
else if (arg == "-ngl" && i + 1 < argc) {
|
||||
n_gpu_layers = std::atoi(argv[++i]);
|
||||
}
|
||||
+ else if (arg == "--t2w-device" && i + 1 < argc) {
|
||||
+ token2wav_device = argv[++i];
|
||||
+ if (token2wav_device != "cpu" && token2wav_device.rfind("gpu:", 0) != 0) {
|
||||
+ fprintf(stderr, "Error: --t2w-device must be cpu or gpu:<index>\n");
|
||||
+ return 1;
|
||||
+ }
|
||||
+ }
|
||||
+ else if (arg == "--t2w-wait-seconds" && i + 1 < argc) {
|
||||
+ token2wav_wait_seconds = std::atoi(argv[++i]);
|
||||
+ if (token2wav_wait_seconds < 1 || token2wav_wait_seconds > 3600) {
|
||||
+ fprintf(stderr, "Error: --t2w-wait-seconds must be 1..3600\n");
|
||||
+ return 1;
|
||||
+ }
|
||||
+ }
|
||||
else if (arg == "--no-tts") {
|
||||
use_tts = false;
|
||||
}
|
||||
@@ -276,6 +304,10 @@ int main(int argc, char ** argv) {
|
||||
}
|
||||
|
||||
// 检查必需参数
|
||||
+ if (max_new_tokens >= n_ctx) {
|
||||
+ fprintf(stderr, "Error: --max-new-tokens must be below ctx-size\n");
|
||||
+ return 1;
|
||||
+ }
|
||||
if (llm_path.empty()) {
|
||||
fprintf(stderr, "Error: -m <llm_model_path> is required\n\n");
|
||||
show_usage(argv[0]);
|
||||
@@ -322,6 +354,9 @@ int main(int argc, char ** argv) {
|
||||
params.vision_coreml_model_path = vision_coreml_model_path;
|
||||
}
|
||||
params.n_ctx = n_ctx;
|
||||
+ if (max_new_tokens > 0) {
|
||||
+ params.n_predict = max_new_tokens;
|
||||
+ }
|
||||
params.n_gpu_layers = n_gpu_layers;
|
||||
|
||||
// Projector 路径需要通过 tts_bin_dir 传递
|
||||
@@ -339,6 +374,7 @@ int main(int argc, char ** argv) {
|
||||
printf(" TTS enabled: %s\n", use_tts ? "yes" : "no");
|
||||
printf(" Context size: %d\n", n_ctx);
|
||||
printf(" GPU layers: %d\n", n_gpu_layers);
|
||||
+ printf(" Token2Wav device: %s\n", token2wav_device.c_str());
|
||||
printf(" Vision backend: %s\n", vision_backend.c_str());
|
||||
if (vision_backend == "coreml") {
|
||||
printf(" Vision CoreML: %s\n", vision_coreml_model_path.c_str());
|
||||
@@ -347,7 +383,8 @@ int main(int argc, char ** argv) {
|
||||
printf(" Ref audio: %s\n", ref_audio_path.c_str());
|
||||
|
||||
// 🔧 Token2Wav 使用 GPU（Metal），已用 ggml_add+ggml_repeat 替代不支持的 ggml_add1
|
||||
- auto ctx_omni = omni_init(¶ms, media_type, use_tts, tts_bin_dir, -1, "gpu:0");
|
||||
+ auto ctx_omni = omni_init(¶ms, media_type, use_tts, tts_bin_dir,
|
||||
+ token2wav_device == "cpu" ? 0 : -1, token2wav_device);
|
||||
if (ctx_omni == nullptr) {
|
||||
fprintf(stderr, "Error: Failed to initialize omni context\n");
|
||||
return 1;
|
||||
@@ -366,13 +403,19 @@ int main(int argc, char ** argv) {
|
||||
}
|
||||
|
||||
// 等待 T2W 完成所有音频生成后再停止线程
|
||||
+ bool audio_complete = true;
|
||||
if(ctx_omni->async && ctx_omni->use_tts) {
|
||||
std::string done_flag = std::string(ctx_omni->base_output_dir) + "/round_000/tts_wav/generation_done.flag";
|
||||
fprintf(stderr, "Waiting for audio generation to complete...\n");
|
||||
- for (int i = 0; i < 1200; ++i) { // 最多等 120 秒
|
||||
+ audio_complete = false;
|
||||
+ for (int i = 0; i < token2wav_wait_seconds * 10; ++i) {
|
||||
FILE * f = fopen(done_flag.c_str(), "r");
|
||||
- if (f) { fclose(f); fprintf(stderr, "Audio generation completed.\n"); break; }
|
||||
- usleep(100000); // 100ms
|
||||
+ if (f) { fclose(f); audio_complete = true; fprintf(stderr, "Audio generation completed.\n"); break; }
|
||||
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
+ }
|
||||
+ if (!audio_complete) {
|
||||
+ fprintf(stderr, "Error: audio generation did not complete within %d seconds.\n",
|
||||
+ token2wav_wait_seconds);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -386,5 +429,8 @@ int main(int argc, char ** argv) {
|
||||
llama_perf_context_print(ctx_omni->ctx_llama);
|
||||
|
||||
omni_free(ctx_omni);
|
||||
+ if (!audio_complete) {
|
||||
+ return 2;
|
||||
+ }
|
||||
return 0;
|
||||
}
|
||||
diff --git a/tools/omni/omni.h b/tools/omni/omni.h
|
||||
index 145f515..b35c459 100644
|
||||
--- a/tools/omni/omni.h
|
||||
+++ b/tools/omni/omni.h
|
||||
@@ -13,7 +13,7 @@
|
||||
#include <atomic>
|
||||
|
||||
// Windows compatibility: pid_t is not defined on MSVC
|
||||
-#ifdef _WIN32
|
||||
+#if defined(_WIN32) && !defined(__MINGW32__)
|
||||
typedef int pid_t;
|
||||
#endif
|
||||
|
||||
@@ -485,4 +485,4 @@ void sliding_window_reset(struct omni_context * ctx_omni);
|
||||
|
||||
// ==================== 高清模式函数声明 ====================
|
||||
// 设置 vision max_slice_nums 覆盖值，用于高清模式
|
||||
-void vision_set_max_slice_nums(struct vision_ctx * ctx_vision, int max_slice_nums);
|
||||
\ No newline at end of file
|
||||
+void vision_set_max_slice_nums(struct vision_ctx * ctx_vision, int max_slice_nums);
|
||||
+469
@@ -0,0 +1,469 @@
|
||||
build: 64 (5202b7b) with x86_64-w64-mingw32-gcc-posix (GCC) 13-posix for x86_64-w64-mingw32
|
||||
llama_model_loader: loaded meta data with 27 key-value pairs and 399 tensors from C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf\MiniCPM-o-4_5-Q4_K_M.gguf (version GGUF V3 (latest))
|
||||
llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
|
||||
llama_model_loader: - kv 0: general.architecture str = qwen3
|
||||
llama_model_loader: - kv 1: general.type str = model
|
||||
llama_model_loader: - kv 2: general.name str = Llm
|
||||
llama_model_loader: - kv 3: general.size_label str = 8.2B
|
||||
llama_model_loader: - kv 4: qwen3.block_count u32 = 36
|
||||
llama_model_loader: - kv 5: qwen3.context_length u32 = 40960
|
||||
llama_model_loader: - kv 6: qwen3.embedding_length u32 = 4096
|
||||
llama_model_loader: - kv 7: qwen3.feed_forward_length u32 = 12288
|
||||
llama_model_loader: - kv 8: qwen3.attention.head_count u32 = 32
|
||||
llama_model_loader: - kv 9: qwen3.attention.head_count_kv u32 = 8
|
||||
llama_model_loader: - kv 10: qwen3.rope.freq_base f32 = 1000000.000000
|
||||
llama_model_loader: - kv 11: qwen3.attention.layer_norm_rms_epsilon f32 = 0.000001
|
||||
llama_model_loader: - kv 12: qwen3.attention.key_length u32 = 128
|
||||
llama_model_loader: - kv 13: qwen3.attention.value_length u32 = 128
|
||||
llama_model_loader: - kv 14: tokenizer.ggml.model str = gpt2
|
||||
llama_model_loader: - kv 15: tokenizer.ggml.pre str = qwen2
|
||||
llama_model_loader: - kv 16: tokenizer.ggml.tokens arr[str,151748] = ["!", "\"", "#", "$", "%", "&", "'", ...
|
||||
llama_model_loader: - kv 17: tokenizer.ggml.token_type arr[i32,151748] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
|
||||
llama_model_loader: - kv 18: tokenizer.ggml.merges arr[str,151387] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
|
||||
llama_model_loader: - kv 19: tokenizer.ggml.bos_token_id u32 = 151644
|
||||
llama_model_loader: - kv 20: tokenizer.ggml.eos_token_id u32 = 151645
|
||||
llama_model_loader: - kv 21: tokenizer.ggml.unknown_token_id u32 = 128244
|
||||
llama_model_loader: - kv 22: tokenizer.ggml.padding_token_id u32 = 151643
|
||||
llama_model_loader: - kv 23: tokenizer.ggml.add_bos_token bool = false
|
||||
llama_model_loader: - kv 24: tokenizer.chat_template str = {%- if tools %}\n {{- '<|im_start|>...
|
||||
llama_model_loader: - kv 25: general.quantization_version u32 = 2
|
||||
llama_model_loader: - kv 26: general.file_type u32 = 15
|
||||
llama_model_loader: - type f32: 145 tensors
|
||||
llama_model_loader: - type q4_K: 217 tensors
|
||||
llama_model_loader: - type q6_K: 37 tensors
|
||||
print_info: file format = GGUF V3 (latest)
|
||||
print_info: file type = Q4_K - Medium
|
||||
print_info: file size = 4.68 GiB (4.90 BPW)
|
||||
load: printing all EOG tokens:
|
||||
load: - 151643 ('<|endoftext|>')
|
||||
load: - 151645 ('<|im_end|>')
|
||||
load: - 151662 ('<|fim_pad|>')
|
||||
load: - 151663 ('<|repo_name|>')
|
||||
load: - 151664 ('<|file_sep|>')
|
||||
load: special tokens cache size = 106
|
||||
load: token to piece cache size = 0.9319 MB
|
||||
print_info: arch = qwen3
|
||||
print_info: vocab_only = 0
|
||||
print_info: n_ctx_train = 40960
|
||||
print_info: n_embd = 4096
|
||||
print_info: n_layer = 36
|
||||
print_info: n_head = 32
|
||||
print_info: n_head_kv = 8
|
||||
print_info: n_rot = 128
|
||||
print_info: n_swa = 0
|
||||
print_info: is_swa_any = 0
|
||||
print_info: n_embd_head_k = 128
|
||||
print_info: n_embd_head_v = 128
|
||||
print_info: n_gqa = 4
|
||||
print_info: n_embd_k_gqa = 1024
|
||||
print_info: n_embd_v_gqa = 1024
|
||||
print_info: f_norm_eps = 0.0e+00
|
||||
print_info: f_norm_rms_eps = 1.0e-06
|
||||
print_info: f_clamp_kqv = 0.0e+00
|
||||
print_info: f_max_alibi_bias = 0.0e+00
|
||||
print_info: f_logit_scale = 0.0e+00
|
||||
print_info: f_attn_scale = 0.0e+00
|
||||
print_info: n_ff = 12288
|
||||
print_info: n_expert = 0
|
||||
print_info: n_expert_used = 0
|
||||
print_info: causal attn = 1
|
||||
print_info: pooling type = -1
|
||||
print_info: rope type = 2
|
||||
print_info: rope scaling = linear
|
||||
print_info: freq_base_train = 1000000.0
|
||||
print_info: freq_scale_train = 1
|
||||
print_info: n_ctx_orig_yarn = 40960
|
||||
print_info: rope_finetuned = unknown
|
||||
print_info: model type = 8B
|
||||
print_info: model params = 8.19 B
|
||||
print_info: general.name = Llm
|
||||
print_info: vocab type = BPE
|
||||
print_info: n_vocab = 151748
|
||||
print_info: n_merges = 151387
|
||||
print_info: BOS token = 151644 '<|im_start|>'
|
||||
print_info: EOS token = 151645 '<|im_end|>'
|
||||
print_info: EOT token = 151645 '<|im_end|>'
|
||||
print_info: UNK token = 128244 '<unk>'
|
||||
print_info: PAD token = 151643 '<|endoftext|>'
|
||||
print_info: LF token = 198 'Ċ'
|
||||
print_info: FIM PRE token = 151659 '<|fim_prefix|>'
|
||||
print_info: FIM SUF token = 151661 '<|fim_suffix|>'
|
||||
print_info: FIM MID token = 151660 '<|fim_middle|>'
|
||||
print_info: FIM PAD token = 151662 '<|fim_pad|>'
|
||||
print_info: FIM REP token = 151663 '<|repo_name|>'
|
||||
print_info: FIM SEP token = 151664 '<|file_sep|>'
|
||||
print_info: EOG token = 151643 '<|endoftext|>'
|
||||
print_info: EOG token = 151645 '<|im_end|>'
|
||||
print_info: EOG token = 151662 '<|fim_pad|>'
|
||||
print_info: EOG token = 151663 '<|repo_name|>'
|
||||
print_info: EOG token = 151664 '<|file_sep|>'
|
||||
print_info: max token length = 256
|
||||
load_tensors: loading model tensors, this can take a while... (mmap = true)
|
||||
load_tensors: CPU_REPACK model buffer size = 3199.50 MiB
|
||||
load_tensors: CPU_Mapped model buffer size = 4761.17 MiB
|
||||
.....................................................................................
|
||||
llama_context: constructing llama_context
|
||||
llama_context: n_seq_max = 1
|
||||
llama_context: n_ctx = 2048
|
||||
llama_context: n_ctx_per_seq = 2048
|
||||
llama_context: n_batch = 2048
|
||||
llama_context: n_ubatch = 512
|
||||
llama_context: causal_attn = 1
|
||||
llama_context: flash_attn = auto
|
||||
llama_context: kv_unified = false
|
||||
llama_context: freq_base = 1000000.0
|
||||
llama_context: freq_scale = 1
|
||||
llama_context: n_ctx_per_seq (2048) < n_ctx_train (40960) -- the full capacity of the model will not be utilized
|
||||
llama_context: CPU output buffer size = 0.58 MiB
|
||||
llama_kv_cache: CPU KV buffer size = 288.00 MiB
|
||||
llama_kv_cache: size = 288.00 MiB ( 2048 cells, 36 layers, 1/1 seqs), K (f16): 144.00 MiB, V (f16): 144.00 MiB
|
||||
llama_context: Flash Attention was auto, set to enabled
|
||||
llama_context: CPU compute buffer size = 304.38 MiB
|
||||
llama_context: graph nodes = 1267
|
||||
llama_context: graph splits = 1
|
||||
llama_model_loader: loaded meta data with 30 key-value pairs and 193 tensors from C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/tts/MiniCPM-o-4_5-tts-F16.gguf (version GGUF V3 (latest))
|
||||
llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
|
||||
llama_model_loader: - kv 0: general.architecture str = llama
|
||||
llama_model_loader: - kv 1: general.type str = model
|
||||
llama_model_loader: - kv 2: general.name str = Tts
|
||||
llama_model_loader: - kv 3: general.size_label str = 445M
|
||||
llama_model_loader: - kv 4: llama.block_count u32 = 20
|
||||
llama_model_loader: - kv 5: llama.context_length u32 = 4096
|
||||
llama_model_loader: - kv 6: llama.embedding_length u32 = 768
|
||||
llama_model_loader: - kv 7: llama.feed_forward_length u32 = 3072
|
||||
llama_model_loader: - kv 8: llama.attention.head_count u32 = 12
|
||||
llama_model_loader: - kv 9: llama.attention.head_count_kv u32 = 12
|
||||
llama_model_loader: - kv 10: llama.rope.freq_base f32 = 10000.000000
|
||||
llama_model_loader: - kv 11: llama.attention.layer_norm_rms_epsilon f32 = 0.000001
|
||||
llama_model_loader: - kv 12: llama.attention.key_length u32 = 64
|
||||
llama_model_loader: - kv 13: llama.attention.value_length u32 = 64
|
||||
llama_model_loader: - kv 14: general.file_type u32 = 1
|
||||
llama_model_loader: - kv 15: llama.vocab_size u32 = 158626
|
||||
llama_model_loader: - kv 16: llama.rope.dimension_count u32 = 64
|
||||
llama_model_loader: - kv 17: general.quantization_version u32 = 2
|
||||
llama_model_loader: - kv 18: tokenizer.ggml.model str = gpt2
|
||||
llama_model_loader: - kv 19: tokenizer.ggml.pre str = qwen2
|
||||
llama_model_loader: - kv 20: tokenizer.ggml.tokens arr[str,158626] = ["!", "\"", "#", "$", "%", "&", "'", ...
|
||||
llama_model_loader: - kv 21: tokenizer.ggml.token_type arr[i32,158626] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
|
||||
llama_model_loader: - kv 22: tokenizer.ggml.merges arr[str,151387] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
|
||||
llama_model_loader: - kv 23: tokenizer.ggml.bos_token_id u32 = 151644
|
||||
llama_model_loader: - kv 24: tokenizer.ggml.eos_token_id u32 = 151645
|
||||
llama_model_loader: - kv 25: tokenizer.ggml.unknown_token_id u32 = 128244
|
||||
llama_model_loader: - kv 26: tokenizer.ggml.padding_token_id u32 = 151643
|
||||
llama_model_loader: - kv 27: tokenizer.ggml.add_bos_token bool = false
|
||||
llama_model_loader: - kv 28: tokenizer.chat_template str = {%- if tools %}\n {{- '<|im_start|>...
|
||||
llama_model_loader: - kv 29: tokenizer.ggml.add_space_prefix bool = false
|
||||
llama_model_loader: - type f32: 50 tensors
|
||||
llama_model_loader: - type f16: 143 tensors
|
||||
print_info: file format = GGUF V3 (latest)
|
||||
print_info: file type = F16
|
||||
print_info: file size = 1.07 GiB (20.70 BPW)
|
||||
load: printing all EOG tokens:
|
||||
load: - 151643 ('<|endoftext|>')
|
||||
load: - 151645 ('<|im_end|>')
|
||||
load: - 151662 ('<|fim_pad|>')
|
||||
load: - 151663 ('<|repo_name|>')
|
||||
load: - 151664 ('<|file_sep|>')
|
||||
load: special tokens cache size = 106
|
||||
load: token to piece cache size = 0.9319 MB
|
||||
print_info: arch = llama
|
||||
print_info: vocab_only = 0
|
||||
print_info: n_ctx_train = 4096
|
||||
print_info: n_embd = 768
|
||||
print_info: n_layer = 20
|
||||
print_info: n_head = 12
|
||||
print_info: n_head_kv = 12
|
||||
print_info: n_rot = 64
|
||||
print_info: n_swa = 0
|
||||
print_info: is_swa_any = 0
|
||||
print_info: n_embd_head_k = 64
|
||||
print_info: n_embd_head_v = 64
|
||||
print_info: n_gqa = 1
|
||||
print_info: n_embd_k_gqa = 768
|
||||
print_info: n_embd_v_gqa = 768
|
||||
print_info: f_norm_eps = 0.0e+00
|
||||
print_info: f_norm_rms_eps = 1.0e-06
|
||||
print_info: f_clamp_kqv = 0.0e+00
|
||||
print_info: f_max_alibi_bias = 0.0e+00
|
||||
print_info: f_logit_scale = 0.0e+00
|
||||
print_info: f_attn_scale = 0.0e+00
|
||||
print_info: n_ff = 3072
|
||||
print_info: n_expert = 0
|
||||
print_info: n_expert_used = 0
|
||||
print_info: causal attn = 1
|
||||
print_info: pooling type = 0
|
||||
print_info: rope type = 0
|
||||
print_info: rope scaling = linear
|
||||
print_info: freq_base_train = 10000.0
|
||||
print_info: freq_scale_train = 1
|
||||
print_info: n_ctx_orig_yarn = 4096
|
||||
print_info: rope_finetuned = unknown
|
||||
print_info: model type = ?B
|
||||
print_info: model params = 444.94 M
|
||||
print_info: general.name = Tts
|
||||
print_info: vocab type = BPE
|
||||
print_info: n_vocab = 158626
|
||||
print_info: n_merges = 151387
|
||||
print_info: BOS token = 151644 '<|im_start|>'
|
||||
print_info: EOS token = 151645 '<|im_end|>'
|
||||
print_info: EOT token = 151645 '<|im_end|>'
|
||||
print_info: UNK token = 128244 '<unk>'
|
||||
print_info: PAD token = 151643 '<|endoftext|>'
|
||||
print_info: LF token = 198 'Ċ'
|
||||
print_info: FIM PRE token = 151659 '<|fim_prefix|>'
|
||||
print_info: FIM SUF token = 151661 '<|fim_suffix|>'
|
||||
print_info: FIM MID token = 151660 '<|fim_middle|>'
|
||||
print_info: FIM PAD token = 151662 '<|fim_pad|>'
|
||||
print_info: FIM REP token = 151663 '<|repo_name|>'
|
||||
print_info: FIM SEP token = 151664 '<|file_sep|>'
|
||||
print_info: EOG token = 151643 '<|endoftext|>'
|
||||
print_info: EOG token = 151645 '<|im_end|>'
|
||||
print_info: EOG token = 151662 '<|fim_pad|>'
|
||||
print_info: EOG token = 151663 '<|repo_name|>'
|
||||
print_info: EOG token = 151664 '<|file_sep|>'
|
||||
print_info: max token length = 256
|
||||
load_tensors: loading model tensors, this can take a while... (mmap = true)
|
||||
load_tensors: skipping TTS tensor emb_code.0.weight (size = 20158464 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor emb_text.weight (size = 467140608 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_semantic.linear1.weight (size = 12582912 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_semantic.linear1.bias (size = 3072 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_semantic.linear2.weight (size = 2359296 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_semantic.linear2.bias (size = 3072 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_spk.linear1.weight (size = 6291456 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_spk.linear1.bias (size = 3072 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_spk.linear2.weight (size = 1179648 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_spk.linear2.bias (size = 3072 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor head_code.0.weight (size = 20158464 bytes) -- will be loaded separately
|
||||
load_tensors: CPU_Mapped model buffer size = 592.48 MiB
|
||||
..............................................................
|
||||
llama_init_from_model: model default pooling_type is [0], but [-1] was specified
|
||||
llama_context: constructing llama_context
|
||||
llama_context: n_seq_max = 1
|
||||
llama_context: n_ctx = 2048
|
||||
llama_context: n_ctx_per_seq = 2048
|
||||
llama_context: n_batch = 2048
|
||||
llama_context: n_ubatch = 512
|
||||
llama_context: causal_attn = 1
|
||||
llama_context: flash_attn = auto
|
||||
llama_context: kv_unified = false
|
||||
llama_context: freq_base = 10000.0
|
||||
llama_context: freq_scale = 1
|
||||
llama_context: n_ctx_per_seq (2048) < n_ctx_train (4096) -- the full capacity of the model will not be utilized
|
||||
llama_context: CPU output buffer size = 0.61 MiB
|
||||
llama_kv_cache: CPU KV buffer size = 120.00 MiB
|
||||
llama_kv_cache: size = 120.00 MiB ( 2048 cells, 20 layers, 1/1 seqs), K (f16): 60.00 MiB, V (f16): 60.00 MiB
|
||||
llama_context: Flash Attention was auto, set to enabled
|
||||
llama_context: CPU compute buffer size = 25.51 MiB
|
||||
llama_context: graph nodes = 613
|
||||
llama_context: graph splits = 1
|
||||
vision_model_loader: model name:
|
||||
vision_model_loader: description: image encoder for MiniCPM-V
|
||||
vision_model_loader: GGUF version: 3
|
||||
vision_model_loader: alignment: 32
|
||||
vision_model_loader: n_tensors: 455
|
||||
vision_model_loader: n_kv: 20
|
||||
|
||||
vision_ctx: vision using CPU backend
|
||||
load_hparams: model_type: 0
|
||||
load_hparams: n_embd: 1152
|
||||
load_hparams: n_head: 16
|
||||
load_hparams: n_ff: 4304
|
||||
load_hparams: n_layer: 27
|
||||
load_hparams: ffn_op: gelu
|
||||
load_hparams: projection_dim: 0
|
||||
|
||||
--- vision hparams ---
|
||||
load_hparams: image_size: 448
|
||||
load_hparams: patch_size: 14
|
||||
load_hparams: minicpmv_version: 100045
|
||||
load_hparams: proj_scale_factor: 0
|
||||
load_hparams: n_wa_pattern: 0
|
||||
|
||||
load_hparams: model size: 1044.36 MiB
|
||||
load_hparams: metadata size: 0.16 MiB
|
||||
load_tensors: ffn up/down are swapped
|
||||
alloc_compute_meta: CPU compute buffer size = 100.30 MiB
|
||||
init tts....init t2w....voc_hg2_model: CPU backend using 8 threads
|
||||
audition_audio_preprocess: Decoded audio - sample_rate=16000, n_samples=32000
|
||||
audition_get_mel_filters: Retrieved mel filters from model: 80 mel bins x 201 fft bins
|
||||
audition_audio_preprocess: Using mel filters - n_mel=80, n_fft=201
|
||||
audition_audio_preprocess: Generated 1 mel spectrogram chunk(s)
|
||||
audition_audio_preprocess: Mel spectrogram ready - n_len=200, n_mel=80, total_size=16000
|
||||
build_whisper: Whisper encoder graph built (projector output, pre-pooling)
|
||||
audition_audio_batch_encode: Audio input data - n_step=200, n_mel=80, total_elements=16000
|
||||
audition_audio_batch_encode: Before incrementing iter: input_frames=200, n_tokens=100, current_iter=0, current_total=0, new_total=100, max_tokens=1500
|
||||
audition_audio_batch_encode: KV cache iter incremented to 1 (total_tokens=100, max_tokens=1500)
|
||||
audition_audio_batch_encode: Final output shape: [4096, 20]
|
||||
audition_audio_batch_encode: Final output: 20 tokens x 4096 dims
|
||||
=== Model Paths ===
|
||||
Base dir: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf
|
||||
LLM: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf\MiniCPM-o-4_5-Q4_K_M.gguf [OK]
|
||||
Vision: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/vision/MiniCPM-o-4_5-vision-F16.gguf [OK]
|
||||
Audio: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/audio/MiniCPM-o-4_5-audio-F16.gguf [OK]
|
||||
TTS: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/tts/MiniCPM-o-4_5-tts-F16.gguf [OK]
|
||||
Projector: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/tts/MiniCPM-o-4_5-projector-F16.gguf [OK]
|
||||
===================
|
||||
=== Initializing Omni Context ===
|
||||
Media type: 2 (omni: audio+vision)
|
||||
TTS enabled: yes
|
||||
Context size: 2048
|
||||
GPU layers: 0
|
||||
Token2Wav device: cpu
|
||||
Vision backend: metal
|
||||
TTS bin dir: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/tts
|
||||
Ref audio: C:\Users\zhout\w2\minicpmo_cpp_cpu_v3\fixture_0000.wav
|
||||
22:55:37.112 === omni_init start
|
||||
22:55:37.112 media_type = 2, duplex_mode = 0, base_output_dir = ./tools/omni/output
|
||||
22:55:37.112 === omni_init: loading new LLM model
|
||||
22:55:39.207 === omni_init: loading TTS model
|
||||
22:55:39.207 TTS model: loading with n_gpu_layers=0
|
||||
22:55:39.457 TTS sampler: temp=0.80, top_p=0.85, top_k=25, rep_penalty=1.05
|
||||
22:55:39.457 TTS: loading weights from GGUF (emb_code, emb_text, projector_semantic, head_code)...
|
||||
22:55:39.781 TTS: head_code shape: dim0=6562, dim1=768, will transpose to [6562, 768]
|
||||
22:55:39.784 TTS: head_code loaded, verifying first few values:
|
||||
22:55:39.784 head_code_weight[0] = 0.01361084 (expect ~0.01385498)
|
||||
22:55:39.784 head_code_weight[1] = -0.01660156 (expect ~-0.01647949)
|
||||
22:55:39.784 head_code_weight[768] = 0.04101562 (this is weight[1, 0], expect ~0.04150391)
|
||||
22:55:39.813 TTS: weights loaded successfully
|
||||
22:55:39.813 Projector: loading from C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/tts/MiniCPM-o-4_5-projector-F16.gguf
|
||||
22:55:39.818 Projector: loaded successfully
|
||||
22:55:39.818 === omni_init: loading APM model
|
||||
22:55:39.992 APM: init from C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/audio/MiniCPM-o-4_5-audio-F16.gguf
|
||||
22:55:40.283 Token2Wav: found models in C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/token2wav-gguf
|
||||
22:55:40.283 Token2Wav: all model files found, initializing session...
|
||||
22:55:40.283 Token2Wav: non-CUDA backend, vocoder using CPU for better performance
|
||||
22:55:40.284 Token2Wav: using prompt_cache from C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/token2wav-gguf/prompt_cache.gguf
|
||||
22:55:40.578 Token2Wav: initialized successfully
|
||||
22:55:40.578 Python T2W: script_dir=./tools/omni/pyt2w, model_dir=./tools/omni/pyt2w/token2wav
|
||||
22:55:40.578 Token2Wav: 使用 C++ 实现
|
||||
22:55:40.578 === omni_init success: ctx_llama = 000001f4ffbf1770
|
||||
=== Running test case ===
|
||||
Audio prefix: C:\Users\zhout\w2\minicpmo_omni_work_cpu\omni-minicpmo-1-gnkn2c28\fixture_
|
||||
Count: 2
|
||||
22:55:40.578 stream_prefill: n_past = 0
|
||||
voice_clone_prompt = <|im_start|>system
|
||||
模仿音频样本的音色并生成新的内容。
|
||||
<|audio_start|>
|
||||
assistant_prompt = <|audio_end|>你的任务是用这种声音模式来当一个助手。请认真、高质量地回复用户的问题。请用高自然度的方式和用户聊天。<|im_end|>
|
||||
<|im_start|>user
|
||||
|
||||
22:55:40.578 system prompt ref_audio: C:\Users\zhout\w2\minicpmo_omni_work_cpu\omni-minicpmo-1-gnkn2c28\fixture_0000.wav
|
||||
22:55:41.684 system prompt ref_audio embedding: n_pos=20
|
||||
22:55:44.068 🔒 n_keep 设置为 72 (system prompt tokens)，这部分永远不会被滑动窗口删除
|
||||
22:55:44.068 stream_prefill(index=0): system prompt 初始化完成，ref_audio 已在其中 prefill
|
||||
22:55:44.068 [SW] system_prompt registered: preserve_length=72 (will be protected from sliding)
|
||||
22:55:44.068 n_past = 72
|
||||
22:55:44.068
|
||||
|
||||
c++ finish stream_prefill(index=0). n_past=72, n_keep=72, n_ctx=2048
|
||||
|
||||
prefill 0 (audio) : 3.48983 s
|
||||
encode_image_with_vision_chunks: step 1 of 1 encoded in 3122.89 ms
|
||||
encode_image_with_vision_chunks: all 1 chunks encoded in 3141.27 ms (grid: 1x1)
|
||||
|
||||
encode_image_with_vision_chunks: image encoded in 3141.27 ms by vision ( 3141.27 ms per chunk, 64 total tokens)
|
||||
omni_image_embed_make_chunks_with_filename: created 1 vision chunks from C:\Users\zhout\w2\minicpmo_omni_work_cpu\omni-minicpmo-1-gnkn2c28\fixture_0001.jpg
|
||||
stream_prefill: prefilled 1 vision chunks (64 tokens each)
|
||||
audition_audio_preprocess: Decoded audio - sample_rate=16000, n_samples=49339
|
||||
audition_audio_preprocess: Audio not aligned to 100ms (49339 samples = 3083.7ms), padded with silence to 49600 samples (3100ms)
|
||||
audition_get_mel_filters: Retrieved mel filters from model: 80 mel bins x 201 fft bins
|
||||
audition_audio_preprocess: Using mel filters - n_mel=80, n_fft=201
|
||||
audition_audio_preprocess: Generated 1 mel spectrogram chunk(s)
|
||||
audition_audio_preprocess: Mel spectrogram ready - n_len=310, n_mel=80, total_size=24800
|
||||
build_whisper: Whisper encoder graph built (projector output, pre-pooling)
|
||||
audition_audio_batch_encode: Audio input data - n_step=310, n_mel=80, total_elements=24800
|
||||
audition_audio_batch_encode: Before incrementing iter: input_frames=310, n_tokens=155, current_iter=1, current_total=155, new_total=310, max_tokens=1500
|
||||
audition_audio_batch_encode: KV cache iter incremented to 2 (total_tokens=310, max_tokens=1500)
|
||||
audition_audio_batch_encode: Final output shape: [4096, 31]
|
||||
audition_audio_batch_encode: Final output: 31 tokens x 4096 dims
|
||||
22:55:50.305 stream_prefill(index=1): processing user audio: C:\Users\zhout\w2\minicpmo_omni_work_cpu\omni-minicpmo-1-gnkn2c28\fixture_0001.wav
|
||||
22:55:50.602 stream_prefill(index=1): user audio embedding: n_pos=31
|
||||
22:55:52.228
|
||||
|
||||
c++ finish stream_prefill(index=1). n_past=172, n_keep=72, n_ctx=2048
|
||||
|
||||
prefill 1 (audio+vision) : 8.15363 s
|
||||
22:55:52.228 📍 stream_decode 开始: n_past=172, n_keep=72, n_ctx=2048, duplex_mode=0
|
||||
22:55:52.228 stream_decode: create tts thread (simplex mode)
|
||||
22:55:52.228 stream_decode: create t2w thread
|
||||
22:55:52.228 wait prefill done
|
||||
22:55:52.229 📍 [单工TTS] 添加 assistant prompt: "<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
<|tts_bos|>", n_past=172
|
||||
22:55:52.229 T2W thread (C++) started
|
||||
<user>
|
||||
<assistant>22:55:52.231 TTS: 创建单工模式输出目录: ./tools/omni/output/round_000
|
||||
22:55:52.232 TTS thread started
|
||||
22:55:52.741 📍 [单工TTS] assistant prompt 完成, n_past=182
|
||||
22:55:52.741 LLM decode: max_tgt_len = 96, n_predict = 96, n_ctx = 2048
|
||||
22:55:53.541 LLM: detected end token, set llm_generation_done=true
|
||||
22:55:53.541 LLM->TTS: text='The square in the image is red.', n_tokens=8, hidden_size=32768, n_embd=4096, token_ids=[785 9334 304 279 2168 374 2518 13]
|
||||
22:55:53.541 📍 Decode 结束: n_past=191, 剩余空间=1857, 无需滑窗
|
||||
22:55:53.541 📍 轮次 1 结束，记录边界于 n_past=191
|
||||
22:55:53.541 TTS<-LLM: chunk_idx=0, text='The square in the image is red.', n_tokens=8, hidden_size=32768, token_ids=[785 9334 304 279 2168 374 2518 13]
|
||||
22:55:53.541 TTS: after queue pop - speek_done=0, llm_finish=1, llm_text.empty=0, token_ids.size=8
|
||||
22:55:53.541 TTS: before empty check - response.empty=0, response='The square in the image is red.', llm_finish=1
|
||||
Waiting for audio generation to complete...
|
||||
TTS: chunk file ./tools/omni/output/round_000/tts_wav/tts_output_chunk_0.wav does not exist or is empty
|
||||
TTS: chunk file ./tools/omni/output/round_000/tts_wav/tts_output_chunk_1.wav does not exist or is empty
|
||||
TTS: no valid WAV files to merge
|
||||
22:55:53.541 TTS: DEBUG before has_llm_data - token_ids.size=8, hidden_states.size=32768, n_embd=4096
|
||||
22:55:53.541 TTS: has_llm_data=1, token_ids=8, hidden_states=32768, n_embd=4096, llm_finish=1
|
||||
22:55:53.542 TTS: n_tokens_orig=8, n_tokens_filtered=8 (filtered 0 special tokens)
|
||||
22:55:53.583 TTS: saved merged embeddings (text) to ./tools/omni/output/round_000/llm_debug/chunk_0/merged_embeddings.txt
|
||||
22:55:53.583 TTS Simplex: generating audio tokens for chunk 0 (n_tokens=8, tts_n_embd=768)
|
||||
22:55:53.583 TTS Simplex: 在 prefill 前添加 audio_bos (chunk_idx=0, new_size=9)
|
||||
22:55:53.583 TTS Simplex: first chunk - cleared KV cache
|
||||
22:55:53.583 TTS Simplex: first chunk - cleared tts_all_generated_tokens and tts_token_buffer
|
||||
22:55:53.688 TTS Simplex: prefill completed, n_past_tts=9
|
||||
22:55:53.688 TTS Simplex: sampler created
|
||||
22:55:53.688 TTS simplex: is_audio_bos=true (first audio token)
|
||||
22:55:53.720 TTS Simplex Phase1: token 1/500: rel_id=761
|
||||
22:55:53.731 TTS Simplex Phase1: token 2/500: rel_id=4218
|
||||
22:55:53.745 TTS Simplex Phase1: token 3/500: rel_id=4218
|
||||
22:55:53.758 TTS Simplex Phase1: token 4/500: rel_id=4137
|
||||
22:55:53.769 TTS Simplex Phase1: token 5/500: rel_id=4137
|
||||
22:55:53.885 📍 为下一轮准备: eval <|im_end|>\n<|im_start|>user\n, n_past=196
|
||||
22:55:54.001 TTS Simplex Phase1: yield 25 tokens 到 T2W (step 25, buffer=25)
|
||||
22:55:54.001 TTS Simplex Phase1: token 25/500: rel_id=3664
|
||||
22:55:54.001 T2W线程(C++): 轮次切换 (-1 -> 0)，更新输出目录
|
||||
22:55:54.001 T2W线程(C++): 新输出目录 ./tools/omni/output/round_000/tts_wav
|
||||
22:55:54.137 TTS Simplex Phase1: EOS token at step 38
|
||||
22:55:54.137 TTS Simplex: is_final_text_chunk=true, will enter Phase 2 for text_eos_embed
|
||||
22:55:54.137 TTS Simplex Phase2: injecting text_eos_embed + audio_bos at n_past=46
|
||||
22:55:54.144 TTS Simplex Phase2: text_eos_embed injected, n_past=47
|
||||
22:55:54.153 TTS Simplex Phase2: audio_bos injected, n_past=48, starting final generation
|
||||
22:55:54.298 TTS Simplex Phase2: yield 25 tokens 到 T2W
|
||||
22:55:54.343 TTS Simplex Phase2: EOS at step 17 (final end)
|
||||
22:55:54.343 TTS Simplex: chunk 结束，tts_token_buffer 剩余 3 tokens (保留等下一个 chunk)
|
||||
22:55:54.343 TTS Simplex: is_final_text_chunk=true, flush 剩余 3 tokens 从 tts_token_buffer
|
||||
22:55:54.343 TTS Simplex: generated 53 audio tokens for chunk 0
|
||||
22:55:54.343 TTS Simplex: chunk 0 first tokens: [761 4218 4218 4137 4137 ], last tokens: [737 136 75 764 735 ]
|
||||
22:55:54.343 TTS Simplex: updated tts_n_past_accumulated=65, total_generated_tokens=53
|
||||
22:55:54.354 TTS: finished processing all chunks
|
||||
22:55:54.354 TTS: 单工模式轮次结束，下一轮 round_idx=1
|
||||
22:55:54.354 TTS: sent is_final=true to T2W queue (turn end)
|
||||
22:55:56.309 🎉 首响时间 (First Audio Response): 4080ms
|
||||
22:55:56.309 T2W线程: wav_0.wav | 0.84s audio | 2306.7ms inference | RTF=2.75 | t=4080ms
|
||||
22:55:58.525 T2W线程: wav_1.wav | 1.00s audio | 2216.0ms inference | RTF=2.22 | t=6296ms
|
||||
22:56:00.521 T2W线程: wav_2.wav | 0.40s audio | 1995.5ms inference | RTF=4.99 | t=8293ms
|
||||
22:56:00.522 T2W线程: 写入结束标记 ./tools/omni/output/round_000/tts_wav/generation_done.flag (last_wav=2)
|
||||
22:56:00.522 T2W线程: 轮次结束后更新输出目录为 ./tools/omni/output/round_001/tts_wav
|
||||
22:56:00.546 omni_stop_threads: stop signals sent
|
||||
22:56:00.546 T2W(C++) 线程: 停止
|
||||
[timing] call=0 tokens=28 final=0 token2mel=1983.849ms vocoder=322.771ms total=2306.633ms audio=20160
|
||||
[timing] call=1 tokens=28 final=0 token2mel=1833.036ms vocoder=382.936ms total=2215.984ms audio=24000
|
||||
[timing] call=2 tokens=6 final=1 token2mel=1846.302ms vocoder=149.170ms total=1995.482ms audio=9600
|
||||
Audio generation completed.
|
||||
llama_perf_context_print: load time = 13532.85 ms
|
||||
llama_perf_context_print: prompt eval time = 12162.64 ms / 182 tokens ( 66.83 ms per token, 14.96 tokens per second)
|
||||
llama_perf_context_print: eval time = 797.52 ms / 9 runs ( 88.61 ms per token, 11.28 tokens per second)
|
||||
llama_perf_context_print: total time = 21337.55 ms / 191 tokens
|
||||
llama_perf_context_print: graphs reused = 10
|
||||
Token2Wav (C++): session released
|
||||
tts thread end
|
||||
t2w thread end
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
build: 64 (5202b7b) with x86_64-w64-mingw32-gcc-posix (GCC) 13-posix for x86_64-w64-mingw32
|
||||
llama_model_loader: loaded meta data with 27 key-value pairs and 399 tensors from C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf\MiniCPM-o-4_5-Q4_K_M.gguf (version GGUF V3 (latest))
|
||||
llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
|
||||
llama_model_loader: - kv 0: general.architecture str = qwen3
|
||||
llama_model_loader: - kv 1: general.type str = model
|
||||
llama_model_loader: - kv 2: general.name str = Llm
|
||||
llama_model_loader: - kv 3: general.size_label str = 8.2B
|
||||
llama_model_loader: - kv 4: qwen3.block_count u32 = 36
|
||||
llama_model_loader: - kv 5: qwen3.context_length u32 = 40960
|
||||
llama_model_loader: - kv 6: qwen3.embedding_length u32 = 4096
|
||||
llama_model_loader: - kv 7: qwen3.feed_forward_length u32 = 12288
|
||||
llama_model_loader: - kv 8: qwen3.attention.head_count u32 = 32
|
||||
llama_model_loader: - kv 9: qwen3.attention.head_count_kv u32 = 8
|
||||
llama_model_loader: - kv 10: qwen3.rope.freq_base f32 = 1000000.000000
|
||||
llama_model_loader: - kv 11: qwen3.attention.layer_norm_rms_epsilon f32 = 0.000001
|
||||
llama_model_loader: - kv 12: qwen3.attention.key_length u32 = 128
|
||||
llama_model_loader: - kv 13: qwen3.attention.value_length u32 = 128
|
||||
llama_model_loader: - kv 14: tokenizer.ggml.model str = gpt2
|
||||
llama_model_loader: - kv 15: tokenizer.ggml.pre str = qwen2
|
||||
+479
@@ -0,0 +1,479 @@
|
||||
build: 64 (5202b7b) with x86_64-w64-mingw32-gcc-posix (GCC) 13-posix for x86_64-w64-mingw32
|
||||
llama_model_load_from_file_impl: using device Vulkan0 (AMD Radeon(TM) 890M Graphics) (unknown id) - 30720 MiB free
|
||||
llama_model_loader: loaded meta data with 27 key-value pairs and 399 tensors from C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf\MiniCPM-o-4_5-Q4_K_M.gguf (version GGUF V3 (latest))
|
||||
llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
|
||||
llama_model_loader: - kv 0: general.architecture str = qwen3
|
||||
llama_model_loader: - kv 1: general.type str = model
|
||||
llama_model_loader: - kv 2: general.name str = Llm
|
||||
llama_model_loader: - kv 3: general.size_label str = 8.2B
|
||||
llama_model_loader: - kv 4: qwen3.block_count u32 = 36
|
||||
llama_model_loader: - kv 5: qwen3.context_length u32 = 40960
|
||||
llama_model_loader: - kv 6: qwen3.embedding_length u32 = 4096
|
||||
llama_model_loader: - kv 7: qwen3.feed_forward_length u32 = 12288
|
||||
llama_model_loader: - kv 8: qwen3.attention.head_count u32 = 32
|
||||
llama_model_loader: - kv 9: qwen3.attention.head_count_kv u32 = 8
|
||||
llama_model_loader: - kv 10: qwen3.rope.freq_base f32 = 1000000.000000
|
||||
llama_model_loader: - kv 11: qwen3.attention.layer_norm_rms_epsilon f32 = 0.000001
|
||||
llama_model_loader: - kv 12: qwen3.attention.key_length u32 = 128
|
||||
llama_model_loader: - kv 13: qwen3.attention.value_length u32 = 128
|
||||
llama_model_loader: - kv 14: tokenizer.ggml.model str = gpt2
|
||||
llama_model_loader: - kv 15: tokenizer.ggml.pre str = qwen2
|
||||
llama_model_loader: - kv 16: tokenizer.ggml.tokens arr[str,151748] = ["!", "\"", "#", "$", "%", "&", "'", ...
|
||||
llama_model_loader: - kv 17: tokenizer.ggml.token_type arr[i32,151748] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
|
||||
llama_model_loader: - kv 18: tokenizer.ggml.merges arr[str,151387] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
|
||||
llama_model_loader: - kv 19: tokenizer.ggml.bos_token_id u32 = 151644
|
||||
llama_model_loader: - kv 20: tokenizer.ggml.eos_token_id u32 = 151645
|
||||
llama_model_loader: - kv 21: tokenizer.ggml.unknown_token_id u32 = 128244
|
||||
llama_model_loader: - kv 22: tokenizer.ggml.padding_token_id u32 = 151643
|
||||
llama_model_loader: - kv 23: tokenizer.ggml.add_bos_token bool = false
|
||||
llama_model_loader: - kv 24: tokenizer.chat_template str = {%- if tools %}\n {{- '<|im_start|>...
|
||||
llama_model_loader: - kv 25: general.quantization_version u32 = 2
|
||||
llama_model_loader: - kv 26: general.file_type u32 = 15
|
||||
llama_model_loader: - type f32: 145 tensors
|
||||
llama_model_loader: - type q4_K: 217 tensors
|
||||
llama_model_loader: - type q6_K: 37 tensors
|
||||
print_info: file format = GGUF V3 (latest)
|
||||
print_info: file type = Q4_K - Medium
|
||||
print_info: file size = 4.68 GiB (4.90 BPW)
|
||||
load: printing all EOG tokens:
|
||||
load: - 151643 ('<|endoftext|>')
|
||||
load: - 151645 ('<|im_end|>')
|
||||
load: - 151662 ('<|fim_pad|>')
|
||||
load: - 151663 ('<|repo_name|>')
|
||||
load: - 151664 ('<|file_sep|>')
|
||||
load: special tokens cache size = 106
|
||||
load: token to piece cache size = 0.9319 MB
|
||||
print_info: arch = qwen3
|
||||
print_info: vocab_only = 0
|
||||
print_info: n_ctx_train = 40960
|
||||
print_info: n_embd = 4096
|
||||
print_info: n_layer = 36
|
||||
print_info: n_head = 32
|
||||
print_info: n_head_kv = 8
|
||||
print_info: n_rot = 128
|
||||
print_info: n_swa = 0
|
||||
print_info: is_swa_any = 0
|
||||
print_info: n_embd_head_k = 128
|
||||
print_info: n_embd_head_v = 128
|
||||
print_info: n_gqa = 4
|
||||
print_info: n_embd_k_gqa = 1024
|
||||
print_info: n_embd_v_gqa = 1024
|
||||
print_info: f_norm_eps = 0.0e+00
|
||||
print_info: f_norm_rms_eps = 1.0e-06
|
||||
print_info: f_clamp_kqv = 0.0e+00
|
||||
print_info: f_max_alibi_bias = 0.0e+00
|
||||
print_info: f_logit_scale = 0.0e+00
|
||||
print_info: f_attn_scale = 0.0e+00
|
||||
print_info: n_ff = 12288
|
||||
print_info: n_expert = 0
|
||||
print_info: n_expert_used = 0
|
||||
print_info: causal attn = 1
|
||||
print_info: pooling type = -1
|
||||
print_info: rope type = 2
|
||||
print_info: rope scaling = linear
|
||||
print_info: freq_base_train = 1000000.0
|
||||
print_info: freq_scale_train = 1
|
||||
print_info: n_ctx_orig_yarn = 40960
|
||||
print_info: rope_finetuned = unknown
|
||||
print_info: model type = 8B
|
||||
print_info: model params = 8.19 B
|
||||
print_info: general.name = Llm
|
||||
print_info: vocab type = BPE
|
||||
print_info: n_vocab = 151748
|
||||
print_info: n_merges = 151387
|
||||
print_info: BOS token = 151644 '<|im_start|>'
|
||||
print_info: EOS token = 151645 '<|im_end|>'
|
||||
print_info: EOT token = 151645 '<|im_end|>'
|
||||
print_info: UNK token = 128244 '<unk>'
|
||||
print_info: PAD token = 151643 '<|endoftext|>'
|
||||
print_info: LF token = 198 'Ċ'
|
||||
print_info: FIM PRE token = 151659 '<|fim_prefix|>'
|
||||
print_info: FIM SUF token = 151661 '<|fim_suffix|>'
|
||||
print_info: FIM MID token = 151660 '<|fim_middle|>'
|
||||
print_info: FIM PAD token = 151662 '<|fim_pad|>'
|
||||
print_info: FIM REP token = 151663 '<|repo_name|>'
|
||||
print_info: FIM SEP token = 151664 '<|file_sep|>'
|
||||
print_info: EOG token = 151643 '<|endoftext|>'
|
||||
print_info: EOG token = 151645 '<|im_end|>'
|
||||
print_info: EOG token = 151662 '<|fim_pad|>'
|
||||
print_info: EOG token = 151663 '<|repo_name|>'
|
||||
print_info: EOG token = 151664 '<|file_sep|>'
|
||||
print_info: max token length = 256
|
||||
load_tensors: loading model tensors, this can take a while... (mmap = true)
|
||||
load_tensors: offloading 36 repeating layers to GPU
|
||||
load_tensors: offloading output layer to GPU
|
||||
load_tensors: offloaded 37/37 layers to GPU
|
||||
load_tensors: Vulkan0 model buffer size = 4454.74 MiB
|
||||
load_tensors: CPU_Mapped model buffer size = 333.43 MiB
|
||||
.....................................................................................
|
||||
llama_context: constructing llama_context
|
||||
llama_context: n_seq_max = 1
|
||||
llama_context: n_ctx = 2048
|
||||
llama_context: n_ctx_per_seq = 2048
|
||||
llama_context: n_batch = 2048
|
||||
llama_context: n_ubatch = 512
|
||||
llama_context: causal_attn = 1
|
||||
llama_context: flash_attn = auto
|
||||
llama_context: kv_unified = false
|
||||
llama_context: freq_base = 1000000.0
|
||||
llama_context: freq_scale = 1
|
||||
llama_context: n_ctx_per_seq (2048) < n_ctx_train (40960) -- the full capacity of the model will not be utilized
|
||||
llama_context: Vulkan_Host output buffer size = 0.58 MiB
|
||||
llama_kv_cache: Vulkan0 KV buffer size = 288.00 MiB
|
||||
llama_kv_cache: size = 288.00 MiB ( 2048 cells, 36 layers, 1/1 seqs), K (f16): 144.00 MiB, V (f16): 144.00 MiB
|
||||
llama_context: Flash Attention was auto, set to enabled
|
||||
llama_context: Vulkan0 compute buffer size = 304.38 MiB
|
||||
llama_context: Vulkan_Host compute buffer size = 12.01 MiB
|
||||
llama_context: graph nodes = 1267
|
||||
llama_context: graph splits = 2
|
||||
llama_model_load_from_file_impl: using device Vulkan0 (AMD Radeon(TM) 890M Graphics) (unknown id) - 25673 MiB free
|
||||
llama_model_loader: loaded meta data with 30 key-value pairs and 193 tensors from C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/tts/MiniCPM-o-4_5-tts-F16.gguf (version GGUF V3 (latest))
|
||||
llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
|
||||
llama_model_loader: - kv 0: general.architecture str = llama
|
||||
llama_model_loader: - kv 1: general.type str = model
|
||||
llama_model_loader: - kv 2: general.name str = Tts
|
||||
llama_model_loader: - kv 3: general.size_label str = 445M
|
||||
llama_model_loader: - kv 4: llama.block_count u32 = 20
|
||||
llama_model_loader: - kv 5: llama.context_length u32 = 4096
|
||||
llama_model_loader: - kv 6: llama.embedding_length u32 = 768
|
||||
llama_model_loader: - kv 7: llama.feed_forward_length u32 = 3072
|
||||
llama_model_loader: - kv 8: llama.attention.head_count u32 = 12
|
||||
llama_model_loader: - kv 9: llama.attention.head_count_kv u32 = 12
|
||||
llama_model_loader: - kv 10: llama.rope.freq_base f32 = 10000.000000
|
||||
llama_model_loader: - kv 11: llama.attention.layer_norm_rms_epsilon f32 = 0.000001
|
||||
llama_model_loader: - kv 12: llama.attention.key_length u32 = 64
|
||||
llama_model_loader: - kv 13: llama.attention.value_length u32 = 64
|
||||
llama_model_loader: - kv 14: general.file_type u32 = 1
|
||||
llama_model_loader: - kv 15: llama.vocab_size u32 = 158626
|
||||
llama_model_loader: - kv 16: llama.rope.dimension_count u32 = 64
|
||||
llama_model_loader: - kv 17: general.quantization_version u32 = 2
|
||||
llama_model_loader: - kv 18: tokenizer.ggml.model str = gpt2
|
||||
llama_model_loader: - kv 19: tokenizer.ggml.pre str = qwen2
|
||||
llama_model_loader: - kv 20: tokenizer.ggml.tokens arr[str,158626] = ["!", "\"", "#", "$", "%", "&", "'", ...
|
||||
llama_model_loader: - kv 21: tokenizer.ggml.token_type arr[i32,158626] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
|
||||
llama_model_loader: - kv 22: tokenizer.ggml.merges arr[str,151387] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",...
|
||||
llama_model_loader: - kv 23: tokenizer.ggml.bos_token_id u32 = 151644
|
||||
llama_model_loader: - kv 24: tokenizer.ggml.eos_token_id u32 = 151645
|
||||
llama_model_loader: - kv 25: tokenizer.ggml.unknown_token_id u32 = 128244
|
||||
llama_model_loader: - kv 26: tokenizer.ggml.padding_token_id u32 = 151643
|
||||
llama_model_loader: - kv 27: tokenizer.ggml.add_bos_token bool = false
|
||||
llama_model_loader: - kv 28: tokenizer.chat_template str = {%- if tools %}\n {{- '<|im_start|>...
|
||||
llama_model_loader: - kv 29: tokenizer.ggml.add_space_prefix bool = false
|
||||
llama_model_loader: - type f32: 50 tensors
|
||||
llama_model_loader: - type f16: 143 tensors
|
||||
print_info: file format = GGUF V3 (latest)
|
||||
print_info: file type = F16
|
||||
print_info: file size = 1.07 GiB (20.70 BPW)
|
||||
load: printing all EOG tokens:
|
||||
load: - 151643 ('<|endoftext|>')
|
||||
load: - 151645 ('<|im_end|>')
|
||||
load: - 151662 ('<|fim_pad|>')
|
||||
load: - 151663 ('<|repo_name|>')
|
||||
load: - 151664 ('<|file_sep|>')
|
||||
load: special tokens cache size = 106
|
||||
load: token to piece cache size = 0.9319 MB
|
||||
print_info: arch = llama
|
||||
print_info: vocab_only = 0
|
||||
print_info: n_ctx_train = 4096
|
||||
print_info: n_embd = 768
|
||||
print_info: n_layer = 20
|
||||
print_info: n_head = 12
|
||||
print_info: n_head_kv = 12
|
||||
print_info: n_rot = 64
|
||||
print_info: n_swa = 0
|
||||
print_info: is_swa_any = 0
|
||||
print_info: n_embd_head_k = 64
|
||||
print_info: n_embd_head_v = 64
|
||||
print_info: n_gqa = 1
|
||||
print_info: n_embd_k_gqa = 768
|
||||
print_info: n_embd_v_gqa = 768
|
||||
print_info: f_norm_eps = 0.0e+00
|
||||
print_info: f_norm_rms_eps = 1.0e-06
|
||||
print_info: f_clamp_kqv = 0.0e+00
|
||||
print_info: f_max_alibi_bias = 0.0e+00
|
||||
print_info: f_logit_scale = 0.0e+00
|
||||
print_info: f_attn_scale = 0.0e+00
|
||||
print_info: n_ff = 3072
|
||||
print_info: n_expert = 0
|
||||
print_info: n_expert_used = 0
|
||||
print_info: causal attn = 1
|
||||
print_info: pooling type = 0
|
||||
print_info: rope type = 0
|
||||
print_info: rope scaling = linear
|
||||
print_info: freq_base_train = 10000.0
|
||||
print_info: freq_scale_train = 1
|
||||
print_info: n_ctx_orig_yarn = 4096
|
||||
print_info: rope_finetuned = unknown
|
||||
print_info: model type = ?B
|
||||
print_info: model params = 444.94 M
|
||||
print_info: general.name = Tts
|
||||
print_info: vocab type = BPE
|
||||
print_info: n_vocab = 158626
|
||||
print_info: n_merges = 151387
|
||||
print_info: BOS token = 151644 '<|im_start|>'
|
||||
print_info: EOS token = 151645 '<|im_end|>'
|
||||
print_info: EOT token = 151645 '<|im_end|>'
|
||||
print_info: UNK token = 128244 '<unk>'
|
||||
print_info: PAD token = 151643 '<|endoftext|>'
|
||||
print_info: LF token = 198 'Ċ'
|
||||
print_info: FIM PRE token = 151659 '<|fim_prefix|>'
|
||||
print_info: FIM SUF token = 151661 '<|fim_suffix|>'
|
||||
print_info: FIM MID token = 151660 '<|fim_middle|>'
|
||||
print_info: FIM PAD token = 151662 '<|fim_pad|>'
|
||||
print_info: FIM REP token = 151663 '<|repo_name|>'
|
||||
print_info: FIM SEP token = 151664 '<|file_sep|>'
|
||||
print_info: EOG token = 151643 '<|endoftext|>'
|
||||
print_info: EOG token = 151645 '<|im_end|>'
|
||||
print_info: EOG token = 151662 '<|fim_pad|>'
|
||||
print_info: EOG token = 151663 '<|repo_name|>'
|
||||
print_info: EOG token = 151664 '<|file_sep|>'
|
||||
print_info: max token length = 256
|
||||
load_tensors: loading model tensors, this can take a while... (mmap = true)
|
||||
load_tensors: skipping TTS tensor emb_code.0.weight (size = 20158464 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor emb_text.weight (size = 467140608 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_semantic.linear1.weight (size = 12582912 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_semantic.linear1.bias (size = 3072 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_semantic.linear2.weight (size = 2359296 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_semantic.linear2.bias (size = 3072 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_spk.linear1.weight (size = 6291456 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_spk.linear1.bias (size = 3072 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_spk.linear2.weight (size = 1179648 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor projector_spk.linear2.bias (size = 3072 bytes) -- will be loaded separately
|
||||
load_tensors: skipping TTS tensor head_code.0.weight (size = 20158464 bytes) -- will be loaded separately
|
||||
load_tensors: offloading 0 repeating layers to GPU
|
||||
load_tensors: offloaded 0/21 layers to GPU
|
||||
load_tensors: CPU_Mapped model buffer size = 592.48 MiB
|
||||
..............................................................
|
||||
llama_init_from_model: model default pooling_type is [0], but [-1] was specified
|
||||
llama_context: constructing llama_context
|
||||
llama_context: n_seq_max = 1
|
||||
llama_context: n_ctx = 2048
|
||||
llama_context: n_ctx_per_seq = 2048
|
||||
llama_context: n_batch = 2048
|
||||
llama_context: n_ubatch = 512
|
||||
llama_context: causal_attn = 1
|
||||
llama_context: flash_attn = auto
|
||||
llama_context: kv_unified = false
|
||||
llama_context: freq_base = 10000.0
|
||||
llama_context: freq_scale = 1
|
||||
llama_context: n_ctx_per_seq (2048) < n_ctx_train (4096) -- the full capacity of the model will not be utilized
|
||||
llama_context: CPU output buffer size = 0.61 MiB
|
||||
llama_kv_cache: CPU KV buffer size = 120.00 MiB
|
||||
llama_kv_cache: size = 120.00 MiB ( 2048 cells, 20 layers, 1/1 seqs), K (f16): 60.00 MiB, V (f16): 60.00 MiB
|
||||
llama_context: Flash Attention was auto, set to enabled
|
||||
llama_context: Vulkan0 compute buffer size = 30.75 MiB
|
||||
llama_context: Vulkan_Host compute buffer size = 7.01 MiB
|
||||
llama_context: graph nodes = 613
|
||||
llama_context: graph splits = 216 (with bs=512), 1 (with bs=1)
|
||||
vision_model_loader: model name:
|
||||
vision_model_loader: description: image encoder for MiniCPM-V
|
||||
vision_model_loader: GGUF version: 3
|
||||
vision_model_loader: alignment: 32
|
||||
vision_model_loader: n_tensors: 455
|
||||
vision_model_loader: n_kv: 20
|
||||
|
||||
vision_ctx: vision using Vulkan0 backend
|
||||
load_hparams: model_type: 0
|
||||
load_hparams: n_embd: 1152
|
||||
load_hparams: n_head: 16
|
||||
load_hparams: n_ff: 4304
|
||||
load_hparams: n_layer: 27
|
||||
load_hparams: ffn_op: gelu
|
||||
load_hparams: projection_dim: 0
|
||||
|
||||
--- vision hparams ---
|
||||
load_hparams: image_size: 448
|
||||
load_hparams: patch_size: 14
|
||||
load_hparams: minicpmv_version: 100045
|
||||
load_hparams: proj_scale_factor: 0
|
||||
load_hparams: n_wa_pattern: 0
|
||||
|
||||
load_hparams: model size: 1044.36 MiB
|
||||
load_hparams: metadata size: 0.16 MiB
|
||||
load_tensors: ffn up/down are swapped
|
||||
alloc_compute_meta: Vulkan0 compute buffer size = 100.30 MiB
|
||||
alloc_compute_meta: CPU compute buffer size = 18.30 MiB
|
||||
init tts....init t2w....voc_hg2_model: CPU backend using 8 threads
|
||||
audition_audio_preprocess: Decoded audio - sample_rate=16000, n_samples=32000
|
||||
audition_get_mel_filters: Retrieved mel filters from model: 80 mel bins x 201 fft bins
|
||||
audition_audio_preprocess: Using mel filters - n_mel=80, n_fft=201
|
||||
audition_audio_preprocess: Generated 1 mel spectrogram chunk(s)
|
||||
audition_audio_preprocess: Mel spectrogram ready - n_len=200, n_mel=80, total_size=16000
|
||||
build_whisper: Whisper encoder graph built (projector output, pre-pooling)
|
||||
audition_audio_batch_encode: Audio input data - n_step=200, n_mel=80, total_elements=16000
|
||||
audition_audio_batch_encode: Before incrementing iter: input_frames=200, n_tokens=100, current_iter=0, current_total=0, new_total=100, max_tokens=1500
|
||||
audition_audio_batch_encode: KV cache iter incremented to 1 (total_tokens=100, max_tokens=1500)
|
||||
audition_audio_batch_encode: Final output shape: [4096, 20]
|
||||
audition_audio_batch_encode: Final output: 20 tokens x 4096 dims
|
||||
=== Model Paths ===
|
||||
Base dir: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf
|
||||
LLM: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf\MiniCPM-o-4_5-Q4_K_M.gguf [OK]
|
||||
Vision: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/vision/MiniCPM-o-4_5-vision-F16.gguf [OK]
|
||||
Audio: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/audio/MiniCPM-o-4_5-audio-F16.gguf [OK]
|
||||
TTS: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/tts/MiniCPM-o-4_5-tts-F16.gguf [OK]
|
||||
Projector: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/tts/MiniCPM-o-4_5-projector-F16.gguf [OK]
|
||||
===================
|
||||
=== Initializing Omni Context ===
|
||||
Media type: 2 (omni: audio+vision)
|
||||
TTS enabled: yes
|
||||
Context size: 2048
|
||||
GPU layers: 99
|
||||
Token2Wav device: cpu
|
||||
Vision backend: metal
|
||||
TTS bin dir: C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/tts
|
||||
Ref audio: C:\Users\zhout\w2\minicpmo_cpp_cpu_v3\fixture_0000.wav
|
||||
22:56:40.659 === omni_init start
|
||||
22:56:40.659 media_type = 2, duplex_mode = 0, base_output_dir = ./tools/omni/output
|
||||
22:56:40.659 === omni_init: loading new LLM model
|
||||
22:56:43.521 === omni_init: loading TTS model
|
||||
22:56:43.521 TTS model: loading with n_gpu_layers=0
|
||||
22:56:43.847 TTS sampler: temp=0.80, top_p=0.85, top_k=25, rep_penalty=1.05
|
||||
22:56:43.847 TTS: loading weights from GGUF (emb_code, emb_text, projector_semantic, head_code)...
|
||||
22:56:44.221 TTS: head_code shape: dim0=6562, dim1=768, will transpose to [6562, 768]
|
||||
22:56:44.224 TTS: head_code loaded, verifying first few values:
|
||||
22:56:44.224 head_code_weight[0] = 0.01361084 (expect ~0.01385498)
|
||||
22:56:44.224 head_code_weight[1] = -0.01660156 (expect ~-0.01647949)
|
||||
22:56:44.224 head_code_weight[768] = 0.04101562 (this is weight[1, 0], expect ~0.04150391)
|
||||
22:56:44.257 TTS: weights loaded successfully
|
||||
22:56:44.257 Projector: loading from C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/tts/MiniCPM-o-4_5-projector-F16.gguf
|
||||
22:56:44.263 Projector: loaded successfully
|
||||
22:56:44.263 === omni_init: loading APM model
|
||||
22:56:44.447 APM: init from C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/audio/MiniCPM-o-4_5-audio-F16.gguf
|
||||
22:56:45.272 Token2Wav: found models in C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/token2wav-gguf
|
||||
22:56:45.272 Token2Wav: all model files found, initializing session...
|
||||
22:56:45.272 Token2Wav: non-CUDA backend, vocoder using CPU for better performance
|
||||
22:56:45.273 Token2Wav: using prompt_cache from C:\Users\zhout\w2\models\MiniCPM-o-4_5-gguf/token2wav-gguf/prompt_cache.gguf
|
||||
22:56:45.545 Token2Wav: initialized successfully
|
||||
22:56:45.545 Python T2W: script_dir=./tools/omni/pyt2w, model_dir=./tools/omni/pyt2w/token2wav
|
||||
22:56:45.545 Token2Wav: 使用 C++ 实现
|
||||
22:56:45.545 === omni_init success: ctx_llama = 00000262404be2d0
|
||||
=== Running test case ===
|
||||
Audio prefix: C:\Users\zhout\w2\minicpmo_omni_work_radeon\omni-minicpmo-1-s53iymbq\fixture_
|
||||
Count: 2
|
||||
22:56:45.545 stream_prefill: n_past = 0
|
||||
voice_clone_prompt = <|im_start|>system
|
||||
模仿音频样本的音色并生成新的内容。
|
||||
<|audio_start|>
|
||||
assistant_prompt = <|audio_end|>你的任务是用这种声音模式来当一个助手。请认真、高质量地回复用户的问题。请用高自然度的方式和用户聊天。<|im_end|>
|
||||
<|im_start|>user
|
||||
|
||||
22:56:45.545 system prompt ref_audio: C:\Users\zhout\w2\minicpmo_omni_work_radeon\omni-minicpmo-1-s53iymbq\fixture_0000.wav
|
||||
22:56:46.248 system prompt ref_audio embedding: n_pos=20
|
||||
22:56:46.947 🔒 n_keep 设置为 72 (system prompt tokens)，这部分永远不会被滑动窗口删除
|
||||
22:56:46.947 stream_prefill(index=0): system prompt 初始化完成，ref_audio 已在其中 prefill
|
||||
22:56:46.947 [SW] system_prompt registered: preserve_length=72 (will be protected from sliding)
|
||||
22:56:46.947 n_past = 72
|
||||
22:56:46.947
|
||||
|
||||
c++ finish stream_prefill(index=0). n_past=72, n_keep=72, n_ctx=2048
|
||||
|
||||
prefill 0 (audio) : 1.40212 s
|
||||
encode_image_with_vision_chunks: step 1 of 1 encoded in 1439.56 ms
|
||||
encode_image_with_vision_chunks: all 1 chunks encoded in 1457.94 ms (grid: 1x1)
|
||||
|
||||
encode_image_with_vision_chunks: image encoded in 1457.94 ms by vision ( 1457.94 ms per chunk, 64 total tokens)
|
||||
omni_image_embed_make_chunks_with_filename: created 1 vision chunks from C:\Users\zhout\w2\minicpmo_omni_work_radeon\omni-minicpmo-1-s53iymbq\fixture_0001.jpg
|
||||
stream_prefill: prefilled 1 vision chunks (64 tokens each)
|
||||
audition_audio_preprocess: Decoded audio - sample_rate=16000, n_samples=49339
|
||||
audition_audio_preprocess: Audio not aligned to 100ms (49339 samples = 3083.7ms), padded with silence to 49600 samples (3100ms)
|
||||
audition_get_mel_filters: Retrieved mel filters from model: 80 mel bins x 201 fft bins
|
||||
audition_audio_preprocess: Using mel filters - n_mel=80, n_fft=201
|
||||
audition_audio_preprocess: Generated 1 mel spectrogram chunk(s)
|
||||
audition_audio_preprocess: Mel spectrogram ready - n_len=310, n_mel=80, total_size=24800
|
||||
build_whisper: Whisper encoder graph built (projector output, pre-pooling)
|
||||
audition_audio_batch_encode: Audio input data - n_step=310, n_mel=80, total_elements=24800
|
||||
audition_audio_batch_encode: Before incrementing iter: input_frames=310, n_tokens=155, current_iter=1, current_total=155, new_total=310, max_tokens=1500
|
||||
audition_audio_batch_encode: KV cache iter incremented to 2 (total_tokens=310, max_tokens=1500)
|
||||
audition_audio_batch_encode: Final output shape: [4096, 31]
|
||||
audition_audio_batch_encode: Final output: 31 tokens x 4096 dims
|
||||
22:56:48.953 stream_prefill(index=1): processing user audio: C:\Users\zhout\w2\minicpmo_omni_work_radeon\omni-minicpmo-1-s53iymbq\fixture_0001.wav
|
||||
22:56:49.244 stream_prefill(index=1): user audio embedding: n_pos=31
|
||||
22:56:49.565
|
||||
|
||||
c++ finish stream_prefill(index=1). n_past=172, n_keep=72, n_ctx=2048
|
||||
|
||||
prefill 1 (audio+vision) : 2.60773 s
|
||||
22:56:49.565 📍 stream_decode 开始: n_past=172, n_keep=72, n_ctx=2048, duplex_mode=0
|
||||
22:56:49.565 stream_decode: create tts thread (simplex mode)
|
||||
22:56:49.565 stream_decode: create t2w thread
|
||||
22:56:49.565 wait prefill done
|
||||
22:56:49.565 📍 [单工TTS] 添加 assistant prompt: "<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
<|tts_bos|>", n_past=172
|
||||
22:56:49.566 T2W thread (C++) started
|
||||
<user>
|
||||
<assistant>22:56:49.568 TTS: 创建单工模式输出目录: ./tools/omni/output/round_000
|
||||
22:56:49.569 TTS thread started
|
||||
22:56:49.728 📍 [单工TTS] assistant prompt 完成, n_past=182
|
||||
22:56:49.728 LLM decode: max_tgt_len = 96, n_predict = 96, n_ctx = 2048
|
||||
22:56:50.241 LLM: detected end token, set llm_generation_done=true
|
||||
22:56:50.241 LLM->TTS: text='The square in the image is red.', n_tokens=8, hidden_size=32768, n_embd=4096, token_ids=[785 9334 304 279 2168 374 2518 13]
|
||||
22:56:50.241 📍 Decode 结束: n_past=191, 剩余空间=1857, 无需滑窗
|
||||
22:56:50.241 📍 轮次 1 结束，记录边界于 n_past=191
|
||||
22:56:50.242 TTS<-LLM: chunk_idx=0, text='The square in the image is red.', n_tokens=8, hidden_size=32768, token_ids=[785 9334 304 279 2168 374 2518 13]
|
||||
22:56:50.242 TTS: after queue pop - speek_done=0, llm_finish=1, llm_text.empty=0, token_ids.size=8
|
||||
22:56:50.242 TTS: before empty check - response.empty=0, response='The square in the image is red.', llm_finish=1
|
||||
Waiting for audio generation to complete...
|
||||
TTS: chunk file ./tools/omni/output/round_000/tts_wav/tts_output_chunk_0.wav does not exist or is empty
|
||||
TTS: chunk file ./tools/omni/output/round_000/tts_wav/tts_output_chunk_1.wav does not exist or is empty
|
||||
TTS: no valid WAV files to merge
|
||||
22:56:50.242 TTS: DEBUG before has_llm_data - token_ids.size=8, hidden_states.size=32768, n_embd=4096
|
||||
22:56:50.242 TTS: has_llm_data=1, token_ids=8, hidden_states=32768, n_embd=4096, llm_finish=1
|
||||
22:56:50.243 TTS: n_tokens_orig=8, n_tokens_filtered=8 (filtered 0 special tokens)
|
||||
22:56:50.278 TTS: saved merged embeddings (text) to ./tools/omni/output/round_000/llm_debug/chunk_0/merged_embeddings.txt
|
||||
22:56:50.278 TTS Simplex: generating audio tokens for chunk 0 (n_tokens=8, tts_n_embd=768)
|
||||
22:56:50.278 TTS Simplex: 在 prefill 前添加 audio_bos (chunk_idx=0, new_size=9)
|
||||
22:56:50.278 TTS Simplex: first chunk - cleared KV cache
|
||||
22:56:50.278 TTS Simplex: first chunk - cleared tts_all_generated_tokens and tts_token_buffer
|
||||
22:56:50.339 TTS Simplex: prefill completed, n_past_tts=9
|
||||
22:56:50.339 TTS Simplex: sampler created
|
||||
22:56:50.339 TTS simplex: is_audio_bos=true (first audio token)
|
||||
22:56:50.364 TTS Simplex Phase1: token 1/500: rel_id=743
|
||||
22:56:50.375 TTS Simplex Phase1: token 2/500: rel_id=4218
|
||||
22:56:50.386 TTS Simplex Phase1: token 3/500: rel_id=4218
|
||||
22:56:50.398 TTS Simplex Phase1: token 4/500: rel_id=4137
|
||||
22:56:50.409 TTS Simplex Phase1: token 5/500: rel_id=4137
|
||||
22:56:50.551 📍 为下一轮准备: eval <|im_end|>\n<|im_start|>user\n, n_past=196
|
||||
22:22:56:56:50.667 TTS Simplex Phase1: yield 25 tokens 到 T2W (step 25, buffer=25)
|
||||
22:56:50.667 TTS Simplex Phase1: token 25/500: rel_id=3670
|
||||
50.667 T2W线程(C++): 轮次切换 (-1 -> 0)，更新输出目录
|
||||
22:56:50.667 T2W线程(C++): 新输出目录 ./tools/omni/output/round_000/tts_wav
|
||||
22:56:50.803 TTS Simplex Phase1: EOS token at step 38
|
||||
22:56:50.803 TTS Simplex: is_final_text_chunk=true, will enter Phase 2 for text_eos_embed
|
||||
22:56:50.803 TTS Simplex Phase2: injecting text_eos_embed + audio_bos at n_past=46
|
||||
22:56:50.812 TTS Simplex Phase2: text_eos_embed injected, n_past=47
|
||||
22:56:50.820 TTS Simplex Phase2: audio_bos injected, n_past=48, starting final generation
|
||||
22:56:50.968 TTS Simplex Phase2: yield 25 tokens 到 T2W
|
||||
22:56:51.119 TTS Simplex Phase2: EOS at step 26 (final end)
|
||||
22:56:51.119 TTS Simplex: chunk 结束，tts_token_buffer 剩余 12 tokens (保留等下一个 chunk)
|
||||
22:56:51.119 TTS Simplex: is_final_text_chunk=true, flush 剩余 12 tokens 从 tts_token_buffer
|
||||
22:56:51.119 TTS Simplex: generated 62 audio tokens for chunk 0
|
||||
22:56:51.119 TTS Simplex: chunk 0 first tokens: [743 4218 4218 4137 4137 ], last tokens: [2919 1488 4218 4218 4137 ]
|
||||
22:56:51.119 TTS Simplex: updated tts_n_past_accumulated=74, total_generated_tokens=62
|
||||
22:56:51.120 TTS: finished processing all chunks
|
||||
22:56:51.120 TTS: 单工模式轮次结束，下一轮 round_idx=1
|
||||
22:56:51.120 TTS: sent is_final=true to T2W queue (turn end)
|
||||
22:56:52.992 🎉 首响时间 (First Audio Response): 3426ms
|
||||
22:56:52.992 T2W线程: wav_0.wav | 0.84s audio | 2323.9ms inference | RTF=2.77 | t=3426ms
|
||||
22:56:55.205 T2W线程: wav_1.wav | 1.00s audio | 2213.0ms inference | RTF=2.21 | t=5639ms
|
||||
22:56:57.330 T2W线程: wav_2.wav | 0.76s audio | 2124.0ms inference | RTF=2.79 | t=7764ms
|
||||
22:56:57.330 T2W线程: 写入结束标记 ./tools/omni/output/round_000/tts_wav/generation_done.flag (last_wav=2)
|
||||
22:56:57.330 T2W线程: 轮次结束后更新输出目录为 ./tools/omni/output/round_001/tts_wav
|
||||
22:56:57.404 omni_stop_threads: stop signals sent
|
||||
22:56:57.404 T2W(C++) 线程: 停止
|
||||
[timing] call=0 tokens=28 final=0 token2mel=1999.902ms vocoder=323.940ms total=2323.871ms audio=20160
|
||||
[timing] call=1 tokens=28 final=0 token2mel=1832.577ms vocoder=380.356ms total=2212.954ms audio=24000
|
||||
[timing] call=2 tokens=15 final=1 token2mel=1868.717ms vocoder=255.300ms total=2124.030ms audio=18240
|
||||
Audio generation completed.
|
||||
llama_perf_context_print: load time = 6067.45 ms
|
||||
llama_perf_context_print: prompt eval time = 4182.74 ms / 182 tokens ( 22.98 ms per token, 43.51 tokens per second)
|
||||
llama_perf_context_print: eval time = 510.14 ms / 9 runs ( 56.68 ms per token, 17.64 tokens per second)
|
||||
llama_perf_context_print: total time = 13742.63 ms / 191 tokens
|
||||
llama_perf_context_print: graphs reused = 10
|
||||
Token2Wav (C++): session released
|
||||
tts thread end
|
||||
t2w thread end
|
||||
+1
@@ -0,0 +1 @@
|
||||
build: 64 (5202b7b) with x86_64-w64-mingw32-gcc-posix (GCC) 13-posix for x86_64-w64-mingw32
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"scope": "MiniCPM-o GGUF complete audio+image\u2192text+speech through Omni StageRuntime/StagePool",
|
||||
"placement": "cpu",
|
||||
"model_revision": "db25077c33951fe163b42986fba0132e279872a2",
|
||||
"artifact_sha256": {
|
||||
"MiniCPM-o-4_5-Q4_K_M.gguf": "1237a97ee081b8abebc47aa7dad565701e8f5f904cdc92f6723ac4281bbc0932",
|
||||
"audio/MiniCPM-o-4_5-audio-F16.gguf": "d5b188ac7feaf98e17175c3f9bd14bf269301bfd187439fdaa3e3a494fc32ef7",
|
||||
"vision/MiniCPM-o-4_5-vision-F16.gguf": "1453678cc4e4fe18de241952962e234f265cb8dda780773526103ab8ba82f421",
|
||||
"tts/MiniCPM-o-4_5-tts-F16.gguf": "c7be3748a863dd6966ae7eed42600b7f41ca67affb03729ff245247f0e5ea088",
|
||||
"tts/MiniCPM-o-4_5-projector-F16.gguf": "4b1b5b377358a5e594a304ff6ea5d52df606a9ba7d886c4299d232f0c67dd1fd",
|
||||
"token2wav-gguf/encoder.gguf": "7f8d265da594eaf5e1de2db8f5f1867dbcb0bb75ef5878fadf2952347116f4d0",
|
||||
"token2wav-gguf/flow_matching.gguf": "eda6069f3edeb5dd3a87fbf2aedb2ddd1b46f3273926c4fcf09b24476a39cab8",
|
||||
"token2wav-gguf/flow_extra.gguf": "c67611aa7d02500fe395a7798bf0bfdfb55c74d37ba93934ca74d82b4e63f78d",
|
||||
"token2wav-gguf/hifigan2.gguf": "1b8b3bf5d8d3066aeee4324fdcdd41aefce170d0ee907645858de408d82835c2",
|
||||
"token2wav-gguf/prompt_cache.gguf": "81fe6f541ebe0b67db06a1e395df928da47991dc9637c2cf47d6c59d5b979f2c"
|
||||
},
|
||||
"cli_sha256": "29247187ddf3b13591cd14cc070e5e25c6bb7547296acc5e8a74e2945811d725",
|
||||
"reference_sha256": "cf4bb5cd9acbfbf67596d4a53310652b61ef4b7ede1d54eb03ef4882ca40223a",
|
||||
"input_sha256": {
|
||||
"audio_wav": "14ee147cbf2ce9afd9e32c8c8228d7cfe11985cd7e63456b2ad33871492aefcc",
|
||||
"image_jpeg": "22c7ccea466a3f551c512045e6316c7597408d4e0a20205502f55784d977aa8d"
|
||||
},
|
||||
"memory_budget": {
|
||||
"capacities": {
|
||||
"host_ram": 32212254720
|
||||
},
|
||||
"demands": {
|
||||
"host_ram": 8589934592
|
||||
}
|
||||
},
|
||||
"profile_count": 1,
|
||||
"warmup_count": 0,
|
||||
"admission_refusal": "ResourceUnavailable: MiniCPM-o weights, state/workspace, I/O and headroom exceed reservation",
|
||||
"status": "passed",
|
||||
"ledger_after_shutdown": {
|
||||
"capacities": {
|
||||
"host_ram": 32212254720
|
||||
},
|
||||
"reserved": {
|
||||
"host_ram": 0
|
||||
},
|
||||
"owners": [],
|
||||
"quarantined": []
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
+153
@@ -0,0 +1,153 @@
|
||||
{
|
||||
"scope": "MiniCPM-o GGUF complete audio+image\u2192text+speech through Omni StageRuntime/StagePool",
|
||||
"placement": "radeon-hybrid",
|
||||
"model_revision": "db25077c33951fe163b42986fba0132e279872a2",
|
||||
"artifact_sha256": {
|
||||
"MiniCPM-o-4_5-Q4_K_M.gguf": "1237a97ee081b8abebc47aa7dad565701e8f5f904cdc92f6723ac4281bbc0932",
|
||||
"audio/MiniCPM-o-4_5-audio-F16.gguf": "d5b188ac7feaf98e17175c3f9bd14bf269301bfd187439fdaa3e3a494fc32ef7",
|
||||
"vision/MiniCPM-o-4_5-vision-F16.gguf": "1453678cc4e4fe18de241952962e234f265cb8dda780773526103ab8ba82f421",
|
||||
"tts/MiniCPM-o-4_5-tts-F16.gguf": "c7be3748a863dd6966ae7eed42600b7f41ca67affb03729ff245247f0e5ea088",
|
||||
"tts/MiniCPM-o-4_5-projector-F16.gguf": "4b1b5b377358a5e594a304ff6ea5d52df606a9ba7d886c4299d232f0c67dd1fd",
|
||||
"token2wav-gguf/encoder.gguf": "7f8d265da594eaf5e1de2db8f5f1867dbcb0bb75ef5878fadf2952347116f4d0",
|
||||
"token2wav-gguf/flow_matching.gguf": "eda6069f3edeb5dd3a87fbf2aedb2ddd1b46f3273926c4fcf09b24476a39cab8",
|
||||
"token2wav-gguf/flow_extra.gguf": "c67611aa7d02500fe395a7798bf0bfdfb55c74d37ba93934ca74d82b4e63f78d",
|
||||
"token2wav-gguf/hifigan2.gguf": "1b8b3bf5d8d3066aeee4324fdcdd41aefce170d0ee907645858de408d82835c2",
|
||||
"token2wav-gguf/prompt_cache.gguf": "81fe6f541ebe0b67db06a1e395df928da47991dc9637c2cf47d6c59d5b979f2c"
|
||||
},
|
||||
"cli_sha256": "66a512386aafeb708f49a7394a9054d94ec777c362395c8917a3d198aef1b523",
|
||||
"reference_sha256": "cf4bb5cd9acbfbf67596d4a53310652b61ef4b7ede1d54eb03ef4882ca40223a",
|
||||
"input_sha256": {
|
||||
"audio_wav": "14ee147cbf2ce9afd9e32c8c8228d7cfe11985cd7e63456b2ad33871492aefcc",
|
||||
"image_jpeg": "22c7ccea466a3f551c512045e6316c7597408d4e0a20205502f55784d977aa8d"
|
||||
},
|
||||
"memory_budget": {
|
||||
"capacities": {
|
||||
"host_ram": 32212254720
|
||||
},
|
||||
"demands": {
|
||||
"host_ram": 25769803776
|
||||
}
|
||||
},
|
||||
"profile_count": 1,
|
||||
"warmup_count": 0,
|
||||
"startup_s": 6.826672600000165,
|
||||
"execution_plan_at_start": {
|
||||
"backend": "external.minicpmo.gguf.v1",
|
||||
"stage_id": 0,
|
||||
"worker_generation": "30587e7aedc54c19be6cd5dd5adbaf0e",
|
||||
"model_revision": "db25077c33951fe163b42986fba0132e279872a2",
|
||||
"artifact_sha256": {
|
||||
"MiniCPM-o-4_5-Q4_K_M.gguf": "1237a97ee081b8abebc47aa7dad565701e8f5f904cdc92f6723ac4281bbc0932",
|
||||
"audio/MiniCPM-o-4_5-audio-F16.gguf": "d5b188ac7feaf98e17175c3f9bd14bf269301bfd187439fdaa3e3a494fc32ef7",
|
||||
"vision/MiniCPM-o-4_5-vision-F16.gguf": "1453678cc4e4fe18de241952962e234f265cb8dda780773526103ab8ba82f421",
|
||||
"tts/MiniCPM-o-4_5-tts-F16.gguf": "c7be3748a863dd6966ae7eed42600b7f41ca67affb03729ff245247f0e5ea088",
|
||||
"tts/MiniCPM-o-4_5-projector-F16.gguf": "4b1b5b377358a5e594a304ff6ea5d52df606a9ba7d886c4299d232f0c67dd1fd",
|
||||
"token2wav-gguf/encoder.gguf": "7f8d265da594eaf5e1de2db8f5f1867dbcb0bb75ef5878fadf2952347116f4d0",
|
||||
"token2wav-gguf/flow_matching.gguf": "eda6069f3edeb5dd3a87fbf2aedb2ddd1b46f3273926c4fcf09b24476a39cab8",
|
||||
"token2wav-gguf/flow_extra.gguf": "c67611aa7d02500fe395a7798bf0bfdfb55c74d37ba93934ca74d82b4e63f78d",
|
||||
"token2wav-gguf/hifigan2.gguf": "1b8b3bf5d8d3066aeee4324fdcdd41aefce170d0ee907645858de408d82835c2",
|
||||
"token2wav-gguf/prompt_cache.gguf": "81fe6f541ebe0b67db06a1e395df928da47991dc9637c2cf47d6c59d5b979f2c"
|
||||
},
|
||||
"cli_sha256": "66a512386aafeb708f49a7394a9054d94ec777c362395c8917a3d198aef1b523",
|
||||
"requested_placement": "radeon-hybrid",
|
||||
"ggml_vk_visible_devices": "1",
|
||||
"shared_ram_reservation": {
|
||||
"host_ram": 25769803776
|
||||
},
|
||||
"weights_bytes": 8872297088,
|
||||
"memory_overhead_bytes": 12884901888,
|
||||
"context_tokens": 2048,
|
||||
"max_new_tokens": 96,
|
||||
"request_capacity": 1,
|
||||
"stateful_session": "C++-owned within one request; discarded after completion",
|
||||
"placement_evidence": "pending first request"
|
||||
},
|
||||
"complete_request": {
|
||||
"text": "The square in the image is red.",
|
||||
"wall_s": 18.274119000067003,
|
||||
"audio_metadata": {
|
||||
"sample_rate_hz": 24000,
|
||||
"channels": 1,
|
||||
"pcm_frames": 62400,
|
||||
"duration_s": 2.6,
|
||||
"pcm_sha256": "412a42f9a72cfaa50e64dfa3f93e9818f37ab3167de31b3d9809e20057440fa3"
|
||||
},
|
||||
"stage_event": {
|
||||
"request_id": "minicpmo-1",
|
||||
"stage_id": 0,
|
||||
"epoch": 1,
|
||||
"seq": 1,
|
||||
"kind": "audio",
|
||||
"worker_generation": "30587e7aedc54c19be6cd5dd5adbaf0e",
|
||||
"buffers": [],
|
||||
"terminal": true,
|
||||
"error": null
|
||||
},
|
||||
"metrics": {
|
||||
"minicpmo_whole_request_wall_s": 18.26119730004575,
|
||||
"peak_sampled_rss_bytes": 15573639168,
|
||||
"worker_log": "C:\\Users\\zhout\\w2\\minicpmo_omni_logs_radeon\\minicpmo-1-fec5accd5f8b463f8e62252d7a31c927.log"
|
||||
}
|
||||
},
|
||||
"output_wav": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\vllm-omni-edge\\benchmarks\\edge_harness\\results\\e2e_expansion_20260923\\evidence\\minicpmo_omni_cpp\\radeon_output.wav",
|
||||
"execution_plan_after_request": {
|
||||
"backend": "external.minicpmo.gguf.v1",
|
||||
"stage_id": 0,
|
||||
"worker_generation": "30587e7aedc54c19be6cd5dd5adbaf0e",
|
||||
"model_revision": "db25077c33951fe163b42986fba0132e279872a2",
|
||||
"artifact_sha256": {
|
||||
"MiniCPM-o-4_5-Q4_K_M.gguf": "1237a97ee081b8abebc47aa7dad565701e8f5f904cdc92f6723ac4281bbc0932",
|
||||
"audio/MiniCPM-o-4_5-audio-F16.gguf": "d5b188ac7feaf98e17175c3f9bd14bf269301bfd187439fdaa3e3a494fc32ef7",
|
||||
"vision/MiniCPM-o-4_5-vision-F16.gguf": "1453678cc4e4fe18de241952962e234f265cb8dda780773526103ab8ba82f421",
|
||||
"tts/MiniCPM-o-4_5-tts-F16.gguf": "c7be3748a863dd6966ae7eed42600b7f41ca67affb03729ff245247f0e5ea088",
|
||||
"tts/MiniCPM-o-4_5-projector-F16.gguf": "4b1b5b377358a5e594a304ff6ea5d52df606a9ba7d886c4299d232f0c67dd1fd",
|
||||
"token2wav-gguf/encoder.gguf": "7f8d265da594eaf5e1de2db8f5f1867dbcb0bb75ef5878fadf2952347116f4d0",
|
||||
"token2wav-gguf/flow_matching.gguf": "eda6069f3edeb5dd3a87fbf2aedb2ddd1b46f3273926c4fcf09b24476a39cab8",
|
||||
"token2wav-gguf/flow_extra.gguf": "c67611aa7d02500fe395a7798bf0bfdfb55c74d37ba93934ca74d82b4e63f78d",
|
||||
"token2wav-gguf/hifigan2.gguf": "1b8b3bf5d8d3066aeee4324fdcdd41aefce170d0ee907645858de408d82835c2",
|
||||
"token2wav-gguf/prompt_cache.gguf": "81fe6f541ebe0b67db06a1e395df928da47991dc9637c2cf47d6c59d5b979f2c"
|
||||
},
|
||||
"cli_sha256": "66a512386aafeb708f49a7394a9054d94ec777c362395c8917a3d198aef1b523",
|
||||
"requested_placement": "radeon-hybrid",
|
||||
"ggml_vk_visible_devices": "1",
|
||||
"shared_ram_reservation": {
|
||||
"host_ram": 25769803776
|
||||
},
|
||||
"weights_bytes": 8872297088,
|
||||
"memory_overhead_bytes": 12884901888,
|
||||
"context_tokens": 2048,
|
||||
"max_new_tokens": 96,
|
||||
"request_capacity": 1,
|
||||
"stateful_session": "C++-owned within one request; discarded after completion",
|
||||
"placement_evidence": {
|
||||
"observed": "radeon-hybrid",
|
||||
"worker_pid": 34076,
|
||||
"peak_sampled_rss_bytes": 15573639168,
|
||||
"worker_log": "C:\\Users\\zhout\\w2\\minicpmo_omni_logs_radeon\\minicpmo-1-fec5accd5f8b463f8e62252d7a31c927.log"
|
||||
}
|
||||
},
|
||||
"abort_check": {
|
||||
"stale_output": false,
|
||||
"ledger": {
|
||||
"capacities": {
|
||||
"host_ram": 32212254720
|
||||
},
|
||||
"reserved": {
|
||||
"host_ram": 0
|
||||
},
|
||||
"owners": [],
|
||||
"quarantined": []
|
||||
}
|
||||
},
|
||||
"status": "passed",
|
||||
"ledger_after_shutdown": {
|
||||
"capacities": {
|
||||
"host_ram": 32212254720
|
||||
},
|
||||
"reserved": {
|
||||
"host_ram": 0
|
||||
},
|
||||
"owners": [],
|
||||
"quarantined": []
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
# E2E check and profiling
|
||||
|
||||
All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [WSL CPU/native Windows expansion](../../../e2e_expansion_20260923/README.md). The expansion adds separate 20-request serial MiniCPM-o text-to-speech profiles on WSL CPU and native Windows GPU, a Spark output-head component profile on the HX370 AMD NPU, complete Spark text-request profiles through standalone llama.cpp and a bounded Omni whole-session stage on native Windows CPU and Radeon 890M, native Windows CPU Qwen3-TTS text-to-WAV profiles through both standalone PyTorch and a bounded Omni whole-session stage, and a hybrid CPU+Radeon 890M Qwen3-TTS profile likewise repeated through a bounded Omni whole-session stage. A separate pinned GGUF MiniCPM-o route adds combined image+audio to text+speech runs on WSL CPU, native Windows CPU and native Windows Radeon 890M+CPU. On the latter two, the same spoken visual question was answered correctly with identical language token IDs. Those C++ routes are standalone, outside Omni. Profiling completion does not imply full release qualification.
|
||||
All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [WSL CPU/native Windows expansion](../../../e2e_expansion_20260923/README.md). The expansion adds separate 20-request serial MiniCPM-o text-to-speech profiles on WSL CPU and native Windows GPU, a Spark output-head component profile on the HX370 AMD NPU, complete Spark text-request profiles through standalone llama.cpp and a bounded Omni whole-session stage on native Windows CPU and Radeon 890M, native Windows CPU Qwen3-TTS text-to-WAV profiles through both standalone PyTorch and a bounded Omni whole-session stage, and a hybrid CPU+Radeon 890M Qwen3-TTS profile likewise repeated through a bounded Omni whole-session stage. A pinned GGUF MiniCPM-o route adds combined image+audio to text+speech runs on WSL CPU, native Windows CPU and native Windows Radeon 890M+CPU. The latter two now also pass one bounded complete request through an Omni graph stage, including in-flight cancellation and ledger release. These are scoped synthetic cases, not full release qualification.
|
||||
|
||||
## Configuration × model
|
||||
|
||||
@@ -9,10 +9,10 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| Device | Spark-X2.5 | Qwen3-TTS 0.6B CustomVoice | Qwen3.8-27B | MiniCPM-o 4.5 | InternVLA-A1 |
|
||||
|---|---|---|---|---|---|
|
||||
| PC HX370: CPU / WSL | profile completed | scoped E2E: short stream | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass |
|
||||
| PC HX370: CPU / native Windows | scoped E2E: Omni Spark text | scoped E2E: Omni CPU text → complete WAV | NOT E2E | scoped E2E: standalone GGUF image+audio → text+speech | synthetic policy pass |
|
||||
| PC HX370: CPU / native Windows | scoped E2E: Omni Spark text | scoped E2E: Omni CPU text → complete WAV | NOT E2E | scoped E2E: Omni GGUF image+audio → text+speech | synthetic policy pass |
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text/image/audio → speech | synthetic policy pass |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | profile completed | profile completed | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass |
|
||||
| PC HX370 + Radeon 890M / Windows worker | scoped E2E: Omni Spark text | scoped E2E: Omni Radeon talker/codec + CPU predictor | NOT E2E | scoped E2E: standalone Radeon+CPU image+audio → text+speech | NOT E2E |
|
||||
| PC HX370 + Radeon 890M / Windows worker | scoped E2E: Omni Spark text | scoped E2E: Omni Radeon talker/codec + CPU predictor | NOT E2E | scoped E2E: Omni Radeon+CPU image+audio → text+speech | NOT E2E |
|
||||
| PC HX370 + AMD NPU / Windows worker | NOT E2E (output-head component passed) | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
|
||||
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
|
||||
@@ -33,7 +33,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC HX370: CPU / native Windows | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The pinned Spark-X2.5-1.7B Q4_K_M GGUF ran in native Windows llama.cpp CPU (`-dev none -ngl 0`) as a whole-session Omni StageRuntime/StagePool backend. `Paris` and the correct `1511` inventory answer passed; one warmup plus 20 serial complete requests were 20/20 correct, nearest-rank wall p50/p95 6.93/7.05 s. The public AsyncOmni short request passed. The stage verifies artifact hashes and CPU placement (0/29 layers offloaded), reserves 4 GiB host RAM, emits a terminal text event and releases reservation after in-flight cancellation. The separate prior standalone profile was 6.65/6.86 s. No vLLM CPU attention repair, incremental stream, restart-after-cancel, loading peak or sustained power/thermal gate is claimed. | Validate broader text/token quality, streaming and cancellation recovery, memory peak and sustained load; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). |
|
||||
| PC HX370: CPU / native Windows | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The pinned 0.6B CustomVoice BF16/SDPA checkpoint ran wholly on CPU through an isolated Qwen wrapper worker owned by one bounded Omni StageRuntime stage. Two named 24 kHz WAVs matched the earlier standalone PCM SHA256 values exactly. One warmup + 20 serial complete Omni requests for 4.56 s of audio had 20/20 identical PCM and nearest-rank wall p50/p95 15.32/15.60 s (RTF 3.36/3.42); independent Whisper tiny.en ASR gave WER 0.20 on the first Omni WAV, an intelligibility proxy only. A public AsyncOmni audio request passed. The stage verified pinned weights/interpreter and CPU placement, reserved 10 GiB host RAM, rejected an insufficient 4 GiB demand before worker launch, and after in-flight cancellation emitted no stale output and returned the ledger to zero. Sampled process-tree private memory peaked at 5.39 GB, not a loading peak. Native vLLM CPU attention operators remain absent; playable streaming, post-cancel restart, broader speech quality, loading peak and sustained power/thermal behavior remain unverified. | Validate playable streaming and restart/recovery, broader voice quality, peak memory, concurrency and sustained behavior; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_omni_cpu/README.md), [standalone baseline](../../../e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md). |
|
||||
| PC HX370: CPU / native Windows | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The installed native Windows vLLM extension lacks cpu_attn_reshape_and_cache and cpu_attn_get_scheduler_metadata. The separate ORT CPU graph worker is not a whole-model text/TTS adapter; a qualified CPU build/binding is missing. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC HX370: CPU / native Windows | MiniCPM-o 4.5 | SCOPED_E2E_STANDALONE_COMPLETE_PATH | The pinned Q4_K_M language and F16 vision/audio/TTS/Token2Wav GGUF set completed a combined synthetic JPEG plus spoken “What color is the square in the image?” input to text and nonzero speech on native Windows CPU. The correct answer, “The square in the image is red.”, used the same eight language token IDs as the Radeon hybrid; independent Whisper tiny.en ASR transcribed both the input and 2.32 s output with WER 0. One cold prompted request took 24.66 s, with 13.11 GB maximum sampled RSS. An earlier unprompted square+tone request produced generic text and 49.16 s of speech in 137.34 s; the first generic build exited 0 with incomplete audio and is preserved as a failure. This is a standalone C++ route, not native vLLM or Omni; one synthetic quality case is not general qualification. The optional upstream WAV merger still has a filename/timing bug, so verified chunks were assembled after completion. | Bind a whole-session native CPU Omni stage with admission/state/cancellation; validate real image/audio suites, playable streaming, loading peak, repeated latency and sustained thermal behavior. [Prompted CPU evidence](../../../e2e_expansion_20260923/evidence/minicpmo_cpp_windows_cpu_prompted/README.md), [unprompted evidence](../../../e2e_expansion_20260923/evidence/minicpmo_cpp_windows_cpu/README.md), [failed first run](../../../e2e_expansion_20260923/evidence/minicpmo_cpp_windows_cpu_attempt1/README.md). |
|
||||
| PC HX370: CPU / native Windows | MiniCPM-o 4.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The pinned Q4_K_M/F16 GGUF set ran one combined synthetic JPEG+spoken-question → text+speech request through a bounded Omni whole-session graph stage on native Windows CPU. It answered “The square in the image is red.” and produced 2.24 s of 24 kHz speech in 24.34 s complete-request wall time (one cold sample, zero warmups); sampled worker RSS peaked at 13.11 GB. The stage verified ten weight hashes and the CLI hash, observed CPU placement, reserved 24 GiB shared RAM, and emitted a terminal audio event. An 8 GiB reservation was refused before worker launch; in-flight cancellation left no stale output and released the ledger. The prior standalone prompted run took 24.66 s and independently transcribed output with WER 0. This is a whole-request C++ worker under Omni control, not native vLLM execution or playable streaming; one synthetic quality case is not general qualification. | Validate real image/audio suites, playable streaming, post-cancel restart, loading peak, repeated latency and sustained thermal behavior. [Omni raw evidence](../../../e2e_expansion_20260923/evidence/minicpmo_omni_cpp/README.md), [standalone prompted evidence](../../../e2e_expansion_20260923/evidence/minicpmo_cpp_windows_cpu_prompted/README.md), [failed first run](../../../e2e_expansion_20260923/evidence/minicpmo_cpp_windows_cpu_attempt1/README.md). |
|
||||
| PC HX370: CPU / native Windows | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded from a UNC path and the direct CPU policy produced finite `[1, 50, 32]` actions on synthetic zero observations/noise. Policy/output CPU placement was checked; one forward took 3.36 s. Relative L2 difference from WSL CPU was 1.08% (no task tolerance). This direct policy route does not require native vLLM CPU attention operators. | Validate real observations/reference actions, units/timestamps, memory and repeated latency before task qualification; [native evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | Spark-X2.5 | PROFILE_COMPLETED | not fully qualified: reference-quality and remaining reliability/performance gates must be reviewed | Broaden only with a named workload/device/precision qualification. |
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | Qwen3-TTS 0.6B CustomVoice | PROFILE_COMPLETED | not fully qualified: reference-quality and remaining reliability/performance gates must be reviewed | Resolve and measure playback stalls; run quality, interruption and thermal tests. |
|
||||
@@ -48,7 +48,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC HX370 + Radeon 890M / Windows worker | Spark-X2.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST | The same pinned Spark-X2.5-1.7B Q4_K_M GGUF completed text prefill/decode through a bounded Omni whole-session stage with llama.cpp Vulkan1. The log identifies Radeon 890M and 29/29 layers offloaded there, rather than RTX 5090; an intentionally wrong device name was refused. `Paris` and `1511` passed; one warmup plus 20 serial complete requests were 20/20 correct, nearest-rank wall p50/p95 2.10/2.14 s. A public AsyncOmni request passed; an oversized context was rejected with a later correct request; in-flight cancellation produced no stale output and released the ledger. The prior standalone run was 2.18/2.20 s. No streaming, restart-after-cancel, GPU-memory peak or sustained power/thermal check is claimed. | Qualify public configuration, broader text/token quality, streaming and cancellation recovery, shared-RAM/GPU peak and sustained use; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/spark_omni_llamacpp/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). |
|
||||
| PC HX370 + Radeon 890M / Windows worker | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_OMNI_COMPLETE_REQUEST_HYBRID | The pinned Q8_0 CustomVoice talker, F16 codec and punctuation GGUFs completed two named text-to-WAV requests through an Omni whole-session CrispASR stage on native Windows. The runtime log identifies Radeon 890M Vulkan0 for talker/codec and the 432 MB code predictor pinned to CPU FP32; this is not all-Radeon or NPU execution. Both PCM hashes exactly match the earlier standalone outputs. Whisper tiny.en transcribed the first Omni WAV and both standalone WAVs exactly (WER 0 as an intelligibility proxy). One warmup plus 20 serial complete Omni requests for 2.56 s of audio had identical PCM and nearest-rank wall p50/p95 3.298/3.336 s (RTF 1.29/1.30). A public AsyncOmni audio request passed; cancellation drained the worker without stale output or remaining reservation. The earlier standalone profile was 3.310/3.345 s. The default Vulkan configuration previously fell back to CPU, all-Vulkan ran away to a 360-frame cap, and DirectML failed waveform quality. No playable stream, post-cancel restart, loading peak, broad speech quality or sustained power/thermal run is claimed. | Validate playable streaming, restart/recovery, broader voice quality, shared-RAM/GPU peak, concurrency and sustained real-time behavior before wider qualification; [Omni raw evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_omni_hybrid/README.md), [standalone evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_radeon890m_joint/README.md), [failed DirectML route](../../../e2e_expansion_20260923/evidence/qwen_tts_radeon890m_directml/README.md). |
|
||||
| PC HX370 + Radeon 890M / Windows worker | Qwen3.8-27B | NOT_E2E_PROFILEABLE | FP32 448x448 vision tower: DirectML 85/113 nodes; normalized max error 9.095e-06 versus torch CPU. Historical component path, not full 27B or current graph-stage image-to-text integration. | Recover the hashed tower bundle and qualify its current Omni graph handoff with the language stage. |
|
||||
| PC HX370 + Radeon 890M / Windows worker | MiniCPM-o 4.5 | SCOPED_E2E_STANDALONE_COMPLETE_PATH_HYBRID | The pinned Q4_K_M/F16 GGUF set completed a **combined spoken question+image → text+speech** path through standalone C++ on native Windows. `GGML_VK_VISIBLE_DEVICES=1` exposed only Radeon 890M; logs show all 37 language layers and vision on Vulkan0, TTS weights at 0/21 GPU layers, and CPU Token2Wav/vocoder. The spoken “What color is the square in the image?” was independently transcribed with WER 0; the model answered “The square in the image is red.” with the same eight language token IDs as CPU, and the 2.56 s WAV was transcribed with WER 0. One cold prompted process took 18.63 s, sampled RSS 15.57 GB. A separate unprompted square+tone run completed 42 WAV chunks/41.96 s audio in 113.43 s. This is CPU+iGPU joint execution, not all-Radeon, NPU or an Omni stage; one synthetic quality case and one cold time are not release qualification. | Bind a full-session Omni stage with explicit shared-RAM/GPU admission and cancellation; validate real multimodal quality, playable streaming, loading peak, repeated/concurrent latency and sustained thermal behavior. [Prompted hybrid evidence](../../../e2e_expansion_20260923/evidence/minicpmo_cpp_radeon890m_prompted/README.md), [unprompted placement and raw run](../../../e2e_expansion_20260923/evidence/minicpmo_cpp_radeon890m/README.md), [CPU parity](../../../e2e_expansion_20260923/evidence/minicpmo_cpp_windows_cpu_prompted/README.md). |
|
||||
| PC HX370 + Radeon 890M / Windows worker | MiniCPM-o 4.5 | SCOPED_E2E_OMNI_COMPLETE_REQUEST_HYBRID | The same pinned GGUF set ran one combined synthetic spoken-question+JPEG → text+speech request through a bounded Omni whole-session graph stage. Logs verify `GGML_VK_VISIBLE_DEVICES=1`, Radeon 890M Vulkan0 language (37/37 layers) and vision, with CPU TTS/Token2Wav/vocoder. It answered “The square in the image is red.” and produced 2.60 s of 24 kHz speech in 18.27 s complete-request wall time (one cold sample, zero warmups); sampled worker RSS peaked at 15.57 GB. The stage verified artifact and binary hashes, reserved 24 GiB shared RAM, emitted a terminal audio event, and in-flight cancellation left no stale output or reservation. The prior standalone prompted run took 18.63 s and its output transcribed with WER 0. This is CPU+iGPU joint execution, not all-Radeon or NPU execution; neither run qualifies general quality or sustained performance. | Validate real multimodal quality, playable streaming, post-cancel restart, loading peak, repeated/concurrent latency and sustained thermal behavior. [Omni raw evidence](../../../e2e_expansion_20260923/evidence/minicpmo_omni_cpp/README.md), [standalone prompted evidence](../../../e2e_expansion_20260923/evidence/minicpmo_cpp_radeon890m_prompted/README.md). |
|
||||
| PC HX370 + Radeon 890M / Windows worker | InternVLA-A1 | NOT_E2E_PROFILEABLE | No placement- and quality-verified whole-model graph bundle/stateful adapter for this model on this AMD route. Small backend graphs do not establish model support. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC HX370 + AMD NPU / Windows worker | Spark-X2.5 | COMPONENT_PASS; NOT_E2E_PROFILEABLE | The real 1.7B output head (RMS norm and full 131,072-logit projection) ran through Omni `external.graph.v1` with VitisAI: one fused NPU partition and seven CPU nodes per request. The A16W8 graph matched top-1 on four captured activations (full-logit SNR 55.43–56.81 dB). One warmup plus 20 serial repeated-input component requests gave p50/p95 10.21/12.44 ms, versus 17.97/20.29 ms for the original FP32 output head on the same Omni CPU stage. NPU startup was 18.21 s versus 4.48 s for FP32 CPU, and NPU worker peak RSS was 3.52 GiB versus 2.06 GiB. No full prefill/decode/KV/sampling, real handoff, power, or sustained run was verified. | Measure an actual vLLM-to-output-head handoff and whole-generation cost, state/cancellation, memory admission and quality before considering a split plan; [component evidence](../../../e2e_expansion_20260923/evidence/spark_amd_npu_output_head/README.md). |
|
||||
| PC HX370 + AMD NPU / Windows worker | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | No placement- and quality-verified whole-model graph bundle/stateful adapter for this model on this AMD route. Small backend graphs do not establish model support. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
|
||||
@@ -89,7 +89,10 @@ from vllm_omni.model_executor.models.ming_tts.pipeline import (
|
||||
MING_TTS_MOE_PIPELINE,
|
||||
MING_TTS_PIPELINE,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO_4_5_PIPELINE
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import (
|
||||
MINICPMO_4_5_GGUF_WHOLE_PIPELINE,
|
||||
MINICPMO_4_5_PIPELINE,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.pipeline import MINIMAX_H3_PIPELINE
|
||||
from vllm_omni.model_executor.models.minimax_music3.pipeline import MINIMAX_MUSIC3_PIPELINE
|
||||
from vllm_omni.model_executor.models.moss_tts.pipeline import (
|
||||
@@ -207,6 +210,7 @@ OMNI_PIPELINES: dict[str, PipelineConfig | PipelineResolverFunc] = {
|
||||
"moss_tts_realtime": MOSS_TTS_REALTIME_PIPELINE,
|
||||
"moss_tts_local": MOSS_TTS_LOCAL_PIPELINE,
|
||||
"minicpmo_4_5": MINICPMO_4_5_PIPELINE,
|
||||
"minicpmo_4_5_gguf_whole": MINICPMO_4_5_GGUF_WHOLE_PIPELINE,
|
||||
"minimax_music3": MINIMAX_MUSIC3_PIPELINE,
|
||||
"higgs_audio_v2": HIGGS_AUDIO_V2_PIPELINE,
|
||||
"higgs_multimodal_qwen3": HIGGS_AUDIO_V3_PIPELINE,
|
||||
|
||||
@@ -21,4 +21,8 @@ def create_graph_client(metadata, config, ledger, reservation):
|
||||
from .qwen_tts_cpu import QwenTTSCPUStageClient
|
||||
|
||||
return QwenTTSCPUStageClient(metadata, backend, ledger, reservation)
|
||||
if name == "external.minicpmo.gguf.v1":
|
||||
from .minicpmo_cpp import MiniCPMOCppStageClient
|
||||
|
||||
return MiniCPMOCppStageClient(metadata, backend, ledger, reservation)
|
||||
raise ValueError(f"unsupported complete-request stage backend: {name!r}")
|
||||
@@ -0,0 +1,496 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Bounded whole-session MiniCPM-o GGUF stage in Omni's graph control path.
|
||||
|
||||
The C++ worker owns weights, KV, TTS and vocoder state for one complete request.
|
||||
Omni owns artifact verification, shared-memory admission, request identity,
|
||||
terminal output acknowledgement and process cancellation. This stage has no
|
||||
incremental playback or state handoff contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import wave
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import psutil
|
||||
import torch
|
||||
from omni_stage_contracts import StageEvent, StageRequest
|
||||
from PIL import Image
|
||||
from vllm.outputs import CompletionOutput
|
||||
|
||||
from vllm_omni.engine.resource_ledger import ResourceUnavailable
|
||||
from vllm_omni.engine.stage_client import StageClientBase
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
_ARTIFACTS = (
|
||||
"MiniCPM-o-4_5-Q4_K_M.gguf",
|
||||
"audio/MiniCPM-o-4_5-audio-F16.gguf",
|
||||
"vision/MiniCPM-o-4_5-vision-F16.gguf",
|
||||
"tts/MiniCPM-o-4_5-tts-F16.gguf",
|
||||
"tts/MiniCPM-o-4_5-projector-F16.gguf",
|
||||
"token2wav-gguf/encoder.gguf",
|
||||
"token2wav-gguf/flow_matching.gguf",
|
||||
"token2wav-gguf/flow_extra.gguf",
|
||||
"token2wav-gguf/hifigan2.gguf",
|
||||
"token2wav-gguf/prompt_cache.gguf",
|
||||
)
|
||||
_WAV_NAME = re.compile(r"wav_(\d+)\.wav")
|
||||
_CHUNK_NAME = re.compile(r"chunk_(\d+)")
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for block in iter(lambda: source.read(8 << 20), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _wav_format(data: bytes, *, sample_rate: int, max_frames: int) -> int:
|
||||
with wave.open(io.BytesIO(data), "rb") as source:
|
||||
fmt = (source.getnchannels(), source.getsampwidth(), source.getframerate())
|
||||
frames = source.getnframes()
|
||||
if fmt != (1, 2, sample_rate) or not 0 < frames <= max_frames:
|
||||
raise ValueError(f"expected bounded mono PCM16 {sample_rate} Hz WAV")
|
||||
if len(source.readframes(frames)) != frames * 2:
|
||||
raise ValueError("truncated input WAV")
|
||||
return frames
|
||||
|
||||
|
||||
def _jpeg_size(data: bytes) -> tuple[int, int]:
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
if image.format != "JPEG":
|
||||
raise ValueError("MiniCPM-o image input must be JPEG")
|
||||
size = image.size
|
||||
image.verify()
|
||||
return size
|
||||
|
||||
|
||||
def _tree_bytes(path: Path) -> int:
|
||||
total = 0
|
||||
for item in path.rglob("*"):
|
||||
try:
|
||||
if item.is_file():
|
||||
total += item.stat().st_size
|
||||
except FileNotFoundError:
|
||||
pass # The worker may atomically replace an output while we sample.
|
||||
return total
|
||||
|
||||
|
||||
def _read_result(work_dir: Path, max_audio_s: float, max_wav_bytes: int,
|
||||
max_text_bytes: int) -> tuple[str, bytes, int]:
|
||||
round_dir = work_dir / "tools" / "omni" / "output" / "round_000"
|
||||
wav_dir = round_dir / "tts_wav"
|
||||
if not (wav_dir / "generation_done.flag").is_file():
|
||||
raise RuntimeError("MiniCPM-o audio generation is incomplete")
|
||||
chunks = sorted(
|
||||
wav_dir.glob("wav_*.wav"),
|
||||
key=lambda path: int(_WAV_NAME.fullmatch(path.name).group(1)),
|
||||
)
|
||||
if not chunks or len(chunks) > 256 or [item.name for item in chunks] != [
|
||||
f"wav_{index}.wav" for index in range(len(chunks))
|
||||
]:
|
||||
raise RuntimeError("MiniCPM-o WAV chunk sequence has gaps or is unbounded")
|
||||
pcm_parts: list[bytes] = []
|
||||
frames = 0
|
||||
for path in chunks:
|
||||
with wave.open(str(path), "rb") as source:
|
||||
fmt = (source.getnchannels(), source.getsampwidth(), source.getframerate())
|
||||
count = source.getnframes()
|
||||
if fmt != (1, 2, 24000) or not 0 < count <= 24000 * 10:
|
||||
raise RuntimeError(f"MiniCPM-o invalid WAV chunk: {path.name}")
|
||||
pcm = source.readframes(count)
|
||||
if len(pcm) != count * 2:
|
||||
raise RuntimeError(f"MiniCPM-o truncated WAV chunk: {path.name}")
|
||||
frames += count
|
||||
if frames > max_audio_s * 24000 or frames * 2 > max_wav_bytes:
|
||||
raise ResourceUnavailable("MiniCPM-o speech exceeds admitted output bound")
|
||||
pcm_parts.append(pcm)
|
||||
pcm = b"".join(pcm_parts)
|
||||
if not np.any(np.frombuffer(pcm, dtype="<i2")):
|
||||
raise RuntimeError("MiniCPM-o generated silent speech")
|
||||
text_paths = sorted(
|
||||
(round_dir / "llm_debug").glob("chunk_*/llm_text.txt"),
|
||||
key=lambda path: int(_CHUNK_NAME.fullmatch(path.parent.name).group(1)),
|
||||
)
|
||||
text = "".join(path.read_text(encoding="utf-8") for path in text_paths).strip()
|
||||
if not text or len(text.encode("utf-8")) > max_text_bytes:
|
||||
raise ResourceUnavailable("MiniCPM-o text is empty or exceeds admitted bound")
|
||||
return text, pcm, frames
|
||||
|
||||
|
||||
class MiniCPMOCppStageClient(StageClientBase):
|
||||
def __init__(self, metadata, config: dict, ledger, reservation) -> None:
|
||||
for name, value in vars(metadata).items():
|
||||
setattr(self, name, value)
|
||||
self.stage_type = "graph"
|
||||
self._ledger, self._reservation = ledger, reservation
|
||||
self._generation = uuid.uuid4().hex
|
||||
self._closed = False
|
||||
self._active: str | None = None
|
||||
self._task: asyncio.Task | None = None
|
||||
self._output: OmniRequestOutput | None = None
|
||||
self._proc: subprocess.Popen | None = None
|
||||
self._lock = threading.Lock()
|
||||
self._epoch = 0
|
||||
self._config = dict(config)
|
||||
self._placement = str(config["placement"])
|
||||
self._memory_pool = str(config.get("memory_pool", "host_ram"))
|
||||
self._context_tokens = int(config.get("context_tokens", 2048))
|
||||
self._max_new_tokens = int(config.get("max_new_tokens", 128))
|
||||
self._max_input_bytes = int(config.get("max_input_bytes", 2 << 20))
|
||||
self._max_wav_bytes = int(config.get("max_wav_bytes", 2 << 20))
|
||||
self._max_text_bytes = int(config.get("max_text_bytes", 16 << 10))
|
||||
self._max_work_bytes = int(config.get("max_work_bytes", 64 << 20))
|
||||
self._max_input_audio_s = float(config.get("max_input_audio_s", 10))
|
||||
self._max_output_audio_s = float(config.get("max_output_audio_s", 20))
|
||||
self._request_timeout_s = float(config.get("request_timeout_s", 300))
|
||||
self._t2w_wait_seconds = int(config.get("t2w_wait_seconds", 180))
|
||||
self._retain_workdir = bool(config.get("retain_workdir", False))
|
||||
try:
|
||||
if self._placement not in ("cpu", "radeon-hybrid"):
|
||||
raise ValueError("MiniCPM-o placement must be cpu or radeon-hybrid")
|
||||
if self._placement == "radeon-hybrid" and (
|
||||
config.get("ggml_vk_visible_devices") != "1"
|
||||
or config.get("expected_gpu_name") != "AMD Radeon(TM) 890M Graphics"
|
||||
):
|
||||
raise ValueError("Radeon stage requires the pinned Vulkan device name and selector")
|
||||
if not (
|
||||
0 < self._max_new_tokens < self._context_tokens
|
||||
and 0 < self._max_input_audio_s <= 30
|
||||
and 0 < self._max_output_audio_s <= 120
|
||||
and 0 < self._max_input_bytes <= 16 << 20
|
||||
and 0 < self._max_wav_bytes <= self._max_work_bytes
|
||||
and 0 < self._max_text_bytes <= self._max_work_bytes
|
||||
and 0 < self._t2w_wait_seconds < self._request_timeout_s <= 3600
|
||||
):
|
||||
raise ValueError("invalid MiniCPM-o context, media, time or output bound")
|
||||
if self._memory_pool not in reservation.demands:
|
||||
raise ResourceUnavailable("MiniCPM-o shared-RAM pool absent from reservation")
|
||||
demand = int(reservation.demands[self._memory_pool])
|
||||
if psutil.virtual_memory().available < demand:
|
||||
raise ResourceUnavailable("available host/shared RAM below MiniCPM-o reservation")
|
||||
self._model_dir = Path(config["model_dir"]).resolve(strict=True)
|
||||
self._binary = Path(config["cli_bin"]).resolve(strict=True)
|
||||
self._reference_wav = Path(config["reference_wav"]).resolve(strict=True)
|
||||
expected = dict(config["artifact_sha256"])
|
||||
if set(expected) != set(_ARTIFACTS):
|
||||
raise ValueError("MiniCPM-o requires the exact ten-artifact manifest")
|
||||
self._artifacts = {name: self._model_dir / name for name in _ARTIFACTS}
|
||||
for name, path in self._artifacts.items():
|
||||
if not path.is_file() or _sha256(path) != str(expected[name]).lower():
|
||||
raise ValueError(f"MiniCPM-o GGUF differs from declared hash: {name}")
|
||||
for path, digest in (
|
||||
(self._binary, str(config["cli_sha256"]).lower()),
|
||||
(self._reference_wav, str(config["reference_sha256"]).lower()),
|
||||
):
|
||||
if _sha256(path) != digest:
|
||||
raise ValueError(f"MiniCPM-o executable/reference differs: {path.name}")
|
||||
_wav_format(self._reference_wav.read_bytes(), sample_rate=16000,
|
||||
max_frames=16000 * 10)
|
||||
overhead = int(config["memory_overhead_bytes"])
|
||||
total_weights = sum(path.stat().st_size for path in self._artifacts.values())
|
||||
if overhead <= 0 or total_weights + overhead + self._max_work_bytes > demand:
|
||||
raise ResourceUnavailable("MiniCPM-o weights, state/workspace, I/O and headroom exceed reservation")
|
||||
help_result = subprocess.run(
|
||||
[str(self._binary), "--help"], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, timeout=10, check=True,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
|
||||
)
|
||||
if not all(flag in help_result.stdout for flag in (
|
||||
b"--max-new-tokens", b"--t2w-device", b"--t2w-wait-seconds"
|
||||
)):
|
||||
raise ValueError("MiniCPM-o CLI lacks bounded-generation or completion support")
|
||||
self._work_root = Path(config["work_root"]).resolve()
|
||||
self._log_root = Path(config["log_root"]).resolve()
|
||||
self._work_root.mkdir(parents=True, exist_ok=True)
|
||||
self._log_root.mkdir(parents=True, exist_ok=True)
|
||||
self.execution_plan = {
|
||||
"backend": "external.minicpmo.gguf.v1",
|
||||
"stage_id": self.stage_id,
|
||||
"worker_generation": self._generation,
|
||||
"model_revision": str(config["model_revision"]),
|
||||
"artifact_sha256": {key: str(value).lower() for key, value in expected.items()},
|
||||
"cli_sha256": str(config["cli_sha256"]).lower(),
|
||||
"requested_placement": self._placement,
|
||||
"ggml_vk_visible_devices": config.get("ggml_vk_visible_devices"),
|
||||
"shared_ram_reservation": dict(reservation.demands),
|
||||
"weights_bytes": total_weights,
|
||||
"memory_overhead_bytes": overhead,
|
||||
"context_tokens": self._context_tokens,
|
||||
"max_new_tokens": self._max_new_tokens,
|
||||
"request_capacity": 1,
|
||||
"stateful_session": "C++-owned within one request; discarded after completion",
|
||||
"placement_evidence": "pending first request",
|
||||
}
|
||||
except BaseException:
|
||||
self.shutdown()
|
||||
raise
|
||||
|
||||
async def add_request_async(self, request_id: str, prompt: Any, params: Any = None) -> None:
|
||||
self.check_health()
|
||||
if self._active is not None:
|
||||
raise ResourceUnavailable("MiniCPM-o stage has an unacknowledged request; capacity is one")
|
||||
if not isinstance(prompt, dict):
|
||||
raise ValueError("MiniCPM-o request must contain audio_wav and image_jpeg bytes")
|
||||
audio, image = prompt.get("audio_wav"), prompt.get("image_jpeg")
|
||||
if not isinstance(audio, (bytes, bytearray)) or not isinstance(image, (bytes, bytearray)):
|
||||
raise ValueError("MiniCPM-o media must use bounded byte buffers")
|
||||
audio, image = bytes(audio), bytes(image)
|
||||
if not 0 < len(audio) <= self._max_input_bytes or not 0 < len(image) <= self._max_input_bytes:
|
||||
raise ResourceUnavailable("MiniCPM-o input exceeds admitted transfer bound")
|
||||
_wav_format(audio, sample_rate=16000,
|
||||
max_frames=int(self._max_input_audio_s * 16000))
|
||||
if _jpeg_size(image) != (448, 448):
|
||||
raise ValueError("MiniCPM-o v1 stage accepts only a 448x448 JPEG")
|
||||
self._epoch += 1
|
||||
request = StageRequest(request_id, self.stage_id, self._epoch, self._generation)
|
||||
self._active = request_id
|
||||
self._task = asyncio.create_task(
|
||||
self._run(request, audio, image), name=f"minicpmo-{request_id}"
|
||||
)
|
||||
|
||||
async def _run(self, request: StageRequest, audio: bytes, image: bytes) -> None:
|
||||
try:
|
||||
result = await asyncio.to_thread(self._execute, request, audio, image)
|
||||
samples = np.frombuffer(result["pcm"], dtype="<i2").astype(np.float32) / 32768.0
|
||||
completion = CompletionOutput(0, result["text"], [], None, None, finish_reason="stop")
|
||||
completion.multimodal_output = {"audio": torch.from_numpy(samples), "sr": 24000}
|
||||
event = StageEvent(
|
||||
request.request_id, self.stage_id, request.epoch, 1, "audio",
|
||||
self._generation, terminal=True,
|
||||
)
|
||||
output = OmniRequestOutput(
|
||||
request_id=request.request_id,
|
||||
prompt="[local audio+image]",
|
||||
stage_id=self.stage_id,
|
||||
final_output_type="audio",
|
||||
outputs=[completion],
|
||||
_custom_output={
|
||||
"stage_event": dataclasses.asdict(event),
|
||||
"audio_metadata": {
|
||||
"sample_rate_hz": 24000,
|
||||
"channels": 1,
|
||||
"pcm_frames": result["frames"],
|
||||
"duration_s": result["frames"] / 24000,
|
||||
"pcm_sha256": hashlib.sha256(result["pcm"]).hexdigest(),
|
||||
},
|
||||
"input_sha256": {
|
||||
"audio": hashlib.sha256(audio).hexdigest(),
|
||||
"image": hashlib.sha256(image).hexdigest(),
|
||||
},
|
||||
},
|
||||
metrics={
|
||||
"minicpmo_whole_request_wall_s": result["wall_s"],
|
||||
"peak_sampled_rss_bytes": result["peak_rss"],
|
||||
"worker_log": str(result["log_path"]),
|
||||
},
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
output = OmniRequestOutput.from_error(request.request_id, str(exc))
|
||||
output.stage_id = self.stage_id
|
||||
if not self._closed and self._epoch == request.epoch:
|
||||
loop = asyncio.get_running_loop()
|
||||
output._stage_release = lambda: loop.call_soon_threadsafe(
|
||||
self.acknowledge, request.request_id, request.epoch, request.worker_generation
|
||||
)
|
||||
self._output = output
|
||||
|
||||
def _execute(self, request: StageRequest, audio: bytes, image: bytes) -> dict:
|
||||
work_dir = Path(tempfile.mkdtemp(prefix=f"omni-minicpmo-{request.epoch}-", dir=self._work_root))
|
||||
if not work_dir.resolve().is_relative_to(self._work_root):
|
||||
raise RuntimeError("MiniCPM-o work directory escaped its declared root")
|
||||
log_path = self._log_root / f"minicpmo-{request.epoch}-{uuid.uuid4().hex}.log"
|
||||
started = time.perf_counter()
|
||||
peak_rss = 0
|
||||
try:
|
||||
shutil.copyfile(self._reference_wav, work_dir / "fixture_0000.wav")
|
||||
(work_dir / "fixture_0001.wav").write_bytes(audio)
|
||||
(work_dir / "fixture_0001.jpg").write_bytes(image)
|
||||
command = [
|
||||
str(self._binary), "-m", str(self._artifacts[_ARTIFACTS[0]]),
|
||||
"-ngl", "0" if self._placement == "cpu" else "99",
|
||||
"-c", str(self._context_tokens),
|
||||
"--max-new-tokens", str(self._max_new_tokens),
|
||||
"--omni", "--t2w-device", "cpu",
|
||||
"--t2w-wait-seconds", str(self._t2w_wait_seconds),
|
||||
"--ref-audio", str(self._reference_wav),
|
||||
"--test", str(work_dir / "fixture_"), "2",
|
||||
]
|
||||
env = os.environ.copy()
|
||||
env["GGML_VK_VISIBLE_DEVICES"] = (
|
||||
"" if self._placement == "cpu" else "1"
|
||||
)
|
||||
with log_path.open("wb") as log:
|
||||
with self._lock:
|
||||
if self._closed or self._epoch != request.epoch:
|
||||
raise RuntimeError("MiniCPM-o request was cancelled before launch")
|
||||
self._proc = subprocess.Popen(
|
||||
command, cwd=work_dir, stdout=log, stderr=subprocess.STDOUT,
|
||||
env=env,
|
||||
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
|
||||
)
|
||||
proc = self._proc
|
||||
process = psutil.Process(proc.pid)
|
||||
deadline = time.monotonic() + self._request_timeout_s
|
||||
while proc.poll() is None:
|
||||
if time.monotonic() >= deadline:
|
||||
self._kill_process(proc)
|
||||
raise TimeoutError("MiniCPM-o complete request timed out")
|
||||
if log_path.stat().st_size + _tree_bytes(work_dir) > self._max_work_bytes:
|
||||
self._kill_process(proc)
|
||||
raise ResourceUnavailable("MiniCPM-o worker exceeded admitted file/I/O bound")
|
||||
try:
|
||||
peak_rss = max(peak_rss, process.memory_info().rss)
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
if peak_rss > self._reservation.demands[self._memory_pool]:
|
||||
self._kill_process(proc)
|
||||
raise ResourceUnavailable("MiniCPM-o worker RSS exceeded shared-RAM reservation")
|
||||
try:
|
||||
proc.wait(timeout=0.1)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"MiniCPM-o CLI exited {proc.returncode}; see {log_path}")
|
||||
log_text = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
if self._placement == "cpu":
|
||||
required = (
|
||||
"GPU layers: 0", "vision_ctx: vision using CPU backend",
|
||||
"Token2Wav device: cpu",
|
||||
)
|
||||
else:
|
||||
required = (
|
||||
"GPU layers: 99",
|
||||
"using device Vulkan0 (AMD Radeon(TM) 890M Graphics)",
|
||||
"offloaded 37/37 layers to GPU",
|
||||
"vision_ctx: vision using Vulkan0 backend",
|
||||
"TTS model: loading with n_gpu_layers=0",
|
||||
"Token2Wav: non-CUDA backend, vocoder using CPU",
|
||||
)
|
||||
if any(item not in log_text for item in required):
|
||||
raise RuntimeError("REFUSE_DEVICE_PLACEMENT: MiniCPM-o did not use declared CPU/Radeon route")
|
||||
if not all(item in log_text for item in (
|
||||
"stream_prefill(index=1): processing user audio:",
|
||||
"stream_prefill: prefilled 1 vision chunks",
|
||||
"Audio generation completed.",
|
||||
)):
|
||||
raise RuntimeError("MiniCPM-o complete input/output path not observed")
|
||||
text, pcm, frames = _read_result(
|
||||
work_dir, self._max_output_audio_s,
|
||||
self._max_wav_bytes, self._max_text_bytes,
|
||||
)
|
||||
self.execution_plan["placement_evidence"] = {
|
||||
"observed": self._placement,
|
||||
"worker_pid": proc.pid,
|
||||
"peak_sampled_rss_bytes": peak_rss,
|
||||
"worker_log": str(log_path),
|
||||
}
|
||||
return {
|
||||
"text": text, "pcm": pcm, "frames": frames,
|
||||
"peak_rss": peak_rss, "wall_s": time.perf_counter() - started,
|
||||
"log_path": log_path,
|
||||
}
|
||||
finally:
|
||||
with self._lock:
|
||||
proc = self._proc
|
||||
if proc is not None:
|
||||
self._kill_process(proc)
|
||||
with self._lock:
|
||||
if self._proc is proc:
|
||||
self._proc = None
|
||||
if not self._retain_workdir:
|
||||
if not work_dir.resolve().is_relative_to(self._work_root):
|
||||
raise RuntimeError("refusing cleanup outside MiniCPM-o work root")
|
||||
shutil.rmtree(work_dir)
|
||||
|
||||
@staticmethod
|
||||
def _kill_process(proc: subprocess.Popen) -> bool:
|
||||
try:
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
except OSError:
|
||||
return False
|
||||
return proc.poll() is not None
|
||||
|
||||
def get_graph_output_nowait(self) -> OmniRequestOutput | None:
|
||||
output, self._output = self._output, None
|
||||
return output
|
||||
|
||||
def acknowledge(self, request_id: str, epoch: int, generation: str) -> None:
|
||||
if (
|
||||
self._active == request_id and epoch == self._epoch
|
||||
and generation == self._generation
|
||||
and (self._task is None or self._task.done())
|
||||
):
|
||||
self._active = None
|
||||
self._task = None
|
||||
|
||||
async def abort_requests_async(self, request_ids: list[str]) -> None:
|
||||
if self._active not in request_ids:
|
||||
return
|
||||
self._epoch += 1
|
||||
self._output = None
|
||||
with self._lock:
|
||||
self._closed = True
|
||||
proc = self._proc
|
||||
drained = await asyncio.to_thread(self._kill_process, proc) if proc else True
|
||||
if self._task is not None and not self._task.done():
|
||||
done, _ = await asyncio.wait({self._task}, timeout=5)
|
||||
if done:
|
||||
self._ledger.release(self._reservation, drained=drained)
|
||||
else:
|
||||
self._ledger.release(self._reservation, drained=False)
|
||||
self._task.add_done_callback(
|
||||
lambda task: self._ledger.release(
|
||||
self._reservation, drained=drained and not task.cancelled()
|
||||
)
|
||||
)
|
||||
self._active = None
|
||||
|
||||
def check_health(self) -> None:
|
||||
if self._closed:
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
|
||||
raise EngineDeadError()
|
||||
|
||||
async def collective_rpc_async(self, method, timeout=None, args=(), kwargs=None):
|
||||
raise NotImplementedError(f"MiniCPM-o backend does not implement collective RPC {method}")
|
||||
|
||||
def shutdown(self) -> None:
|
||||
with self._lock:
|
||||
self._closed = True
|
||||
proc = self._proc
|
||||
drained = self._kill_process(proc) if proc else True
|
||||
self._output = None
|
||||
task = self._task
|
||||
if task is not None and not task.done():
|
||||
self._ledger.release(self._reservation, drained=False)
|
||||
task.add_done_callback(
|
||||
lambda done: self._ledger.release(
|
||||
self._reservation, drained=drained and not done.cancelled()
|
||||
)
|
||||
)
|
||||
else:
|
||||
self._ledger.release(self._reservation, drained=drained)
|
||||
@@ -87,3 +87,21 @@ MINICPMO_4_5_PIPELINE = PipelineConfig(
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# The GGUF C++ worker owns the complete autoregressive/vision/audio/TTS session.
|
||||
# This is a separate whole-session deployment of MiniCPM-o, not a replacement
|
||||
# for the three-stage vLLM checkpoint above. Omni's graph path controls one
|
||||
# bounded request, resource reservation and terminal audio acknowledgement.
|
||||
MINICPMO_4_5_GGUF_WHOLE_PIPELINE = PipelineConfig(
|
||||
model_type="minicpmo_4_5_gguf_whole",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="audio",
|
||||
execution_type=StageExecutionType.GRAPH,
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
),
|
||||
),
|
||||
)
|
||||
Reference in new issue
Block a user