Record MiniCPM-o CPU GGUF complete-path evidence
This commit is contained in:
1 parent
bb8735effc
commit
445ca92ef1
190 files changed
+4625
-5
No files matched your search
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check generated Mandarin speech against its own text with local Whisper.
|
||||
|
||||
Character error rate is an intelligibility proxy, not a task-quality or
|
||||
speaker-fidelity score. It does not say whether the model understood its input.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(8 << 20), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def characters(value: str) -> list[str]:
|
||||
return re.findall(r"[\u4e00-\u9fff]|[a-z0-9]", value.casefold())
|
||||
|
||||
|
||||
def character_error_rate(reference: str, actual: str) -> float:
|
||||
expected, got = characters(reference), characters(actual)
|
||||
previous = list(range(len(got) + 1))
|
||||
for position, item in enumerate(expected, 1):
|
||||
current = [position]
|
||||
for index, candidate in enumerate(got, 1):
|
||||
current.append(min(
|
||||
current[-1] + 1,
|
||||
previous[index] + 1,
|
||||
previous[index - 1] + (item != candidate),
|
||||
))
|
||||
previous = current
|
||||
return previous[-1] / max(len(expected), 1)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--wav", type=Path, required=True)
|
||||
parser.add_argument("--reference-file", type=Path, required=True)
|
||||
parser.add_argument("--model-dir", type=Path, required=True)
|
||||
parser.add_argument("--model-revision", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
from opencc import OpenCC
|
||||
import soundfile as sf
|
||||
import torch
|
||||
import transformers
|
||||
from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
|
||||
|
||||
audio, sample_rate = sf.read(args.wav, dtype="float32", always_2d=False)
|
||||
if audio.ndim != 1 or not audio.size or not np.isfinite(audio).all():
|
||||
raise ValueError("expected finite mono audio")
|
||||
audio_16k = librosa.resample(audio, orig_sr=sample_rate, target_sr=16000)
|
||||
reference = args.reference_file.read_text(encoding="utf-8").strip()
|
||||
if not characters(reference):
|
||||
raise ValueError("reference text has no Mandarin or alphanumeric characters")
|
||||
started = time.perf_counter()
|
||||
processor = AutoProcessor.from_pretrained(str(args.model_dir), local_files_only=True)
|
||||
model = AutoModelForSpeechSeq2Seq.from_pretrained(
|
||||
str(args.model_dir), local_files_only=True
|
||||
).to("cpu")
|
||||
load_s = time.perf_counter() - started
|
||||
if {parameter.device.type for parameter in model.parameters()} != {"cpu"}:
|
||||
raise RuntimeError("Whisper was not wholly on CPU")
|
||||
started = time.perf_counter()
|
||||
transcripts = []
|
||||
segment_frames = 25 * 16000
|
||||
with torch.no_grad():
|
||||
for offset in range(0, len(audio_16k), segment_frames):
|
||||
features = processor(
|
||||
audio_16k[offset:offset + segment_frames],
|
||||
sampling_rate=16000,
|
||||
return_tensors="pt",
|
||||
return_attention_mask=True,
|
||||
)
|
||||
ids = model.generate(
|
||||
features.input_features, attention_mask=features.attention_mask,
|
||||
language="zh", task="transcribe",
|
||||
)
|
||||
transcripts.append(
|
||||
processor.batch_decode(ids, skip_special_tokens=True)[0].strip()
|
||||
)
|
||||
infer_s = time.perf_counter() - started
|
||||
transcript = "".join(transcripts)
|
||||
traditional_to_simplified = OpenCC("t2s")
|
||||
normalized_reference = traditional_to_simplified.convert(reference)
|
||||
normalized_transcript = traditional_to_simplified.convert(transcript)
|
||||
report = {
|
||||
"scope": "Whisper Mandarin ASR proxy for generated speech; no input-task or perceptual-quality claim",
|
||||
"asr_model": "openai/whisper-tiny",
|
||||
"asr_revision": args.model_revision,
|
||||
"asr_weights_sha256": sha256(args.model_dir / "model.safetensors"),
|
||||
"transformers": transformers.__version__,
|
||||
"torch": torch.__version__,
|
||||
"wav_sha256": sha256(args.wav),
|
||||
"wav_duration_s": len(audio) / sample_rate,
|
||||
"reference_text": reference,
|
||||
"transcript": transcript,
|
||||
"asr_segments_25_s": len(transcripts),
|
||||
"character_error_rate_raw": character_error_rate(reference, transcript),
|
||||
"character_error_rate_t2s_normalized": character_error_rate(
|
||||
normalized_reference, normalized_transcript
|
||||
),
|
||||
"normalization": "OpenCC 0.1.7 t2s; punctuation ignored",
|
||||
"load_s": load_s,
|
||||
"inference_s": infer_s,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capture one local process's raw output, exit code, elapsed time and memory.
|
||||
|
||||
This wrapper samples RSS and private bytes; the values are observed maxima, not
|
||||
proof of the true loading peak between samples.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import psutil
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--cwd", type=Path, required=True)
|
||||
parser.add_argument("--stdout", type=Path, required=True)
|
||||
parser.add_argument("--stderr", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
parser.add_argument("--sample-interval-ms", type=int, default=100)
|
||||
parser.add_argument("command", nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
if not args.command or args.sample_interval_ms < 10:
|
||||
parser.error("a command and sample interval >= 10 ms are required")
|
||||
command = args.command[1:] if args.command[0] == "--" else args.command
|
||||
for path in (args.stdout, args.stderr, args.report):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
started = time.perf_counter()
|
||||
with args.stdout.open("wb") as stdout, args.stderr.open("wb") as stderr:
|
||||
child = subprocess.Popen(command, cwd=args.cwd, stdout=stdout, stderr=stderr)
|
||||
process = psutil.Process(child.pid)
|
||||
samples = 0
|
||||
peak_rss = 0
|
||||
peak_private = 0
|
||||
while child.poll() is None:
|
||||
try:
|
||||
memory = process.memory_info()
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
break
|
||||
peak_rss = max(peak_rss, memory.rss)
|
||||
peak_private = max(peak_private, getattr(memory, "private", 0))
|
||||
samples += 1
|
||||
time.sleep(args.sample_interval_ms / 1000)
|
||||
exit_code = child.wait()
|
||||
report = {
|
||||
"command": command,
|
||||
"cwd": str(args.cwd),
|
||||
"exit_code": exit_code,
|
||||
"wall_s": time.perf_counter() - started,
|
||||
"memory_sample_interval_ms": args.sample_interval_ms,
|
||||
"memory_samples": samples,
|
||||
"peak_sampled_rss_bytes": peak_rss,
|
||||
"peak_sampled_private_bytes": peak_private,
|
||||
"measurement_note": "sampled process values, not guaranteed loading peaks",
|
||||
}
|
||||
args.report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2))
|
||||
if exit_code:
|
||||
raise SystemExit(exit_code)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify a complete MiniCPM-o C++ text/audio result and join its PCM chunks.
|
||||
|
||||
The upstream CLI currently writes wav_N.wav but its optional merger searches
|
||||
for tts_output_chunk_N.wav. This verifier checks the actual bounded sequence,
|
||||
then joins PCM frames without changing sample values.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import array
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ARTIFACTS = (
|
||||
"MiniCPM-o-4_5-Q4_K_M.gguf",
|
||||
"audio/MiniCPM-o-4_5-audio-F16.gguf",
|
||||
"vision/MiniCPM-o-4_5-vision-F16.gguf",
|
||||
"tts/MiniCPM-o-4_5-tts-F16.gguf",
|
||||
"tts/MiniCPM-o-4_5-projector-F16.gguf",
|
||||
"token2wav-gguf/encoder.gguf",
|
||||
"token2wav-gguf/flow_matching.gguf",
|
||||
"token2wav-gguf/flow_extra.gguf",
|
||||
"token2wav-gguf/hifigan2.gguf",
|
||||
"token2wav-gguf/prompt_cache.gguf",
|
||||
)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(8 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--run-log", type=Path, required=True)
|
||||
parser.add_argument("--runtime-bin", type=Path, required=True)
|
||||
parser.add_argument("--model-dir", type=Path, required=True)
|
||||
parser.add_argument("--model-revision", required=True)
|
||||
parser.add_argument("--input-image", type=Path, required=True)
|
||||
parser.add_argument("--input-audio", type=Path, required=True)
|
||||
parser.add_argument("--merged-wav", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path, required=True)
|
||||
parser.add_argument("--platform-label", default="WSL CPU")
|
||||
parser.add_argument("--process-profile", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
output_dir = args.output_dir.resolve(strict=True)
|
||||
log = args.run_log.read_text(encoding="utf-8", errors="replace")
|
||||
required_log = (
|
||||
"Media type: 2 (omni: audio+vision)",
|
||||
"GPU layers: 0",
|
||||
"Token2Wav device: cpu",
|
||||
"vision_ctx: vision using CPU backend",
|
||||
"Token2Wav: initialized successfully",
|
||||
"stream_prefill(index=1): processing user audio:",
|
||||
"stream_prefill(index=1): user audio embedding:",
|
||||
"stream_prefill: prefilled 1 vision chunks",
|
||||
)
|
||||
missing_log = [item for item in required_log if item not in log]
|
||||
if missing_log:
|
||||
raise RuntimeError(f"missing complete-path/placement evidence: {missing_log}")
|
||||
process_profile = None
|
||||
if args.process_profile:
|
||||
process_profile = json.loads(args.process_profile.read_text(encoding="utf-8"))
|
||||
if process_profile["exit_code"] != 0:
|
||||
raise RuntimeError("profiled process exited with an error")
|
||||
elif "Exit status: 0" not in log:
|
||||
raise RuntimeError("/usr/bin/time did not report exit status 0")
|
||||
wave_dir = output_dir / "round_000" / "tts_wav"
|
||||
if not (wave_dir / "generation_done.flag").is_file():
|
||||
raise RuntimeError("Token2Wav did not finish")
|
||||
chunk_paths = sorted(
|
||||
wave_dir.glob("wav_*.wav"),
|
||||
key=lambda path: int(re.fullmatch(r"wav_(\d+)\.wav", path.name).group(1)),
|
||||
)
|
||||
if not 0 < len(chunk_paths) <= 256:
|
||||
raise RuntimeError("no bounded audio chunk sequence")
|
||||
if [path.name for path in chunk_paths] != [f"wav_{i}.wav" for i in range(len(chunk_paths))]:
|
||||
raise RuntimeError("audio chunk sequence has a gap or duplicate")
|
||||
|
||||
pcm_chunks: list[bytes] = []
|
||||
chunks: list[dict] = []
|
||||
square_sum = 0
|
||||
frames_total = 0
|
||||
for path in chunk_paths:
|
||||
with wave.open(str(path), "rb") as source:
|
||||
fmt = (source.getnchannels(), source.getsampwidth(), source.getframerate())
|
||||
if fmt != (1, 2, 24000):
|
||||
raise RuntimeError(f"invalid PCM16 mono 24 kHz chunk: {path.name}, {fmt}")
|
||||
frames = source.getnframes()
|
||||
if not 0 < frames <= 24000 * 10:
|
||||
raise RuntimeError(f"invalid bounded chunk length: {path.name}, {frames}")
|
||||
pcm = source.readframes(frames)
|
||||
if len(pcm) != frames * 2:
|
||||
raise RuntimeError(f"truncated audio chunk: {path.name}")
|
||||
samples = array.array("h")
|
||||
samples.frombytes(pcm)
|
||||
square_sum += sum(int(value) ** 2 for value in samples)
|
||||
frames_total += frames
|
||||
pcm_chunks.append(pcm)
|
||||
chunks.append({"name": path.name, "frames": frames, "sha256": sha256(path)})
|
||||
if frames_total > 24000 * 120 or square_sum == 0:
|
||||
raise RuntimeError("assembled audio is too long or silent")
|
||||
pcm = b"".join(pcm_chunks)
|
||||
args.merged_wav.parent.mkdir(parents=True, exist_ok=True)
|
||||
with wave.open(str(args.merged_wav), "wb") as target:
|
||||
target.setnchannels(1)
|
||||
target.setsampwidth(2)
|
||||
target.setframerate(24000)
|
||||
target.writeframes(pcm)
|
||||
|
||||
text_dir = output_dir / "round_000" / "llm_debug"
|
||||
text_paths = sorted(
|
||||
text_dir.glob("chunk_*/llm_text.txt"),
|
||||
key=lambda path: int(path.parent.name.removeprefix("chunk_")),
|
||||
)
|
||||
response_text = "".join(path.read_text(encoding="utf-8") for path in text_paths).strip()
|
||||
if not response_text:
|
||||
raise RuntimeError("language stage returned no text")
|
||||
memory_match = re.search(r"Maximum resident set size \(kbytes\): (\d+)", log)
|
||||
wall_match = re.search(r"Elapsed \(wall clock\) time \(h:mm:ss or m:ss\): ([\d:.]+)", log)
|
||||
model_dir = args.model_dir.resolve(strict=True)
|
||||
artifacts = {}
|
||||
for name in ARTIFACTS:
|
||||
path = model_dir / name
|
||||
artifacts[name] = {"bytes": path.stat().st_size, "sha256": sha256(path)}
|
||||
report = {
|
||||
"scope": f"standalone MiniCPM-o 4.5 GGUF audio+image input to text+speech on {args.platform_label}",
|
||||
"status": "scoped_complete_path_pass",
|
||||
"quality": "synthetic red-square/tone response not established as semantically correct",
|
||||
"model_revision": args.model_revision,
|
||||
"artifacts": artifacts,
|
||||
"runtime_binary_sha256": sha256(args.runtime_bin),
|
||||
"input_image_sha256": sha256(args.input_image),
|
||||
"input_audio_sha256": sha256(args.input_audio),
|
||||
"response_text": response_text,
|
||||
"audio_chunks": chunks,
|
||||
"audio_frames": frames_total,
|
||||
"audio_duration_s": frames_total / 24000,
|
||||
"audio_rms_pcm16": math.sqrt(square_sum / frames_total),
|
||||
"pcm_sha256": hashlib.sha256(pcm).hexdigest(),
|
||||
"merged_wav_sha256": sha256(args.merged_wav),
|
||||
"peak_rss_kib_from_time": int(memory_match.group(1)) if memory_match else None,
|
||||
"wall_clock_from_time": wall_match.group(1) if wall_match else None,
|
||||
"sampled_process_profile": process_profile,
|
||||
"upstream_merge_name_mismatch": "TTS: no valid WAV files to merge" in log,
|
||||
}
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
print(json.dumps({key: value for key, value in report.items() if key != "artifacts"}, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user