diff --git a/benchmarks/edge_harness/experiments/probe_qwen_tts_native_cpu.py b/benchmarks/edge_harness/experiments/probe_qwen_tts_native_cpu.py new file mode 100644 index 000000000..bd3738983 --- /dev/null +++ b/benchmarks/edge_harness/experiments/probe_qwen_tts_native_cpu.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Run the pinned Qwen3-TTS CustomVoice checkpoint on native Windows CPU. + +This uses Qwen's PyTorch wrapper as a standalone backend feasibility check. +It does not prove Omni streaming, playback, or speech intelligibility. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import platform +import time +from pathlib import Path + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--output-wav", type=Path, required=True) + parser.add_argument("--output-report", type=Path, required=True) + parser.add_argument("--text", default="Hello from the local computer.") + parser.add_argument("--speaker", default="Ryan") + parser.add_argument("--max-new-tokens", type=int, default=64) + parser.add_argument("--threads", type=int, default=8) + parser.add_argument("--warmups", type=int, default=0) + parser.add_argument("--repeats", type=int, default=1) + args = parser.parse_args() + if args.max_new_tokens <= 0 or args.threads <= 0 or args.warmups < 0 or args.repeats <= 0: + parser.error("Token/thread/repeat counts must be positive and warmups nonnegative") + checkpoint = args.model_dir / "model.safetensors" + tokenizer_weights = args.model_dir / "speech_tokenizer" / "model.safetensors" + if not checkpoint.is_file() or not tokenizer_weights.is_file(): + raise FileNotFoundError("The exact talker and speech-tokenizer weights are required") + + os.environ["HF_HUB_OFFLINE"] = "1" + os.environ["TRANSFORMERS_OFFLINE"] = "1" + import numpy as np + import soundfile as sf + import torch + import transformers + from qwen_tts import Qwen3TTSModel + + torch.set_num_threads(args.threads) + started = time.perf_counter() + model = Qwen3TTSModel.from_pretrained( + str(args.model_dir.resolve()), + device_map="cpu", + dtype=torch.bfloat16, + attn_implementation="sdpa", + local_files_only=True, + ) + startup_s = time.perf_counter() - started + model_devices = sorted({parameter.device.type for parameter in model.model.parameters()}) + if model_devices != ["cpu"]: + raise RuntimeError(f"TTS model was not wholly on CPU: {model_devices}") + if args.speaker.lower() not in model.get_supported_speakers(): + raise ValueError("The requested speaker is not supported by this checkpoint") + + def generate_one(*, save_wav: bool) -> dict: + torch.manual_seed(42) + started = time.perf_counter() + wavs, sample_rate = model.generate_custom_voice( + text=args.text, + language="English", + speaker=args.speaker, + max_new_tokens=args.max_new_tokens, + do_sample=True, + non_streaming_mode=True, + ) + request_s = time.perf_counter() - started + if len(wavs) != 1 or sample_rate <= 0: + raise RuntimeError("Expected one waveform with a valid sample rate") + wav = np.asarray(wavs[0], dtype=np.float32).reshape(-1) + if not wav.size or not np.isfinite(wav).all(): + raise RuntimeError("TTS returned an empty or non-finite waveform") + rms = float(np.sqrt(np.mean(wav.astype(np.float64) ** 2))) + if rms <= 1e-5: + raise RuntimeError("TTS waveform is effectively silent") + if save_wav: + args.output_wav.parent.mkdir(parents=True, exist_ok=True) + sf.write(args.output_wav, wav, sample_rate, subtype="PCM_16") + return { + "request_s": request_s, + "sample_rate": sample_rate, + "samples": int(wav.size), + "duration_s": float(wav.size / sample_rate), + "rms": rms, + "peak_abs": float(np.max(np.abs(wav))), + "nonzero_samples": int(np.count_nonzero(wav)), + "float32_sha256": hashlib.sha256(wav.tobytes()).hexdigest(), + } + + warmups = [generate_one(save_wav=False) for _ in range(args.warmups)] + measured = [generate_one(save_wav=index == 0) for index in range(args.repeats)] + first = measured[0] + latencies = sorted(row["request_s"] for row in measured) + report = { + "scope": "one complete native Windows CPU text-to-WAV request via Qwen's standalone PyTorch wrapper; no Omni streaming or speech-quality claim", + "platform": platform.platform(), + "python": platform.python_version(), + "torch": torch.__version__, + "transformers": transformers.__version__, + "model_dir": str(args.model_dir.resolve()), + "talker_sha256": _sha256(checkpoint), + "speech_tokenizer_sha256": _sha256(tokenizer_weights), + "model_devices": model_devices, + "dtype": "bfloat16", + "attention": "sdpa", + "threads": args.threads, + "text": args.text, + "speaker": args.speaker, + "language": "English", + "max_new_tokens": args.max_new_tokens, + "seed": 42, + "startup_s": startup_s, + "request_s": first["request_s"], + "sample_rate": first["sample_rate"], + "samples": first["samples"], + "duration_s": first["duration_s"], + "rms": first["rms"], + "peak_abs": first["peak_abs"], + "nonzero_samples": first["nonzero_samples"], + "wav_sha256": _sha256(args.output_wav), + "warmups": warmups, + "measured": measured, + "profile": { + "repeats": args.repeats, + "concurrency": 1, + "p50_request_s": latencies[math.ceil(0.5 * len(latencies)) - 1], + "p95_request_s": latencies[math.ceil(0.95 * len(latencies)) - 1], + "all_same_float32_audio": len({row["float32_sha256"] for row in measured}) == 1, + }, + "status": "passed", + } + args.output_report.parent.mkdir(parents=True, exist_ok=True) + args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/edge_harness/experiments/probe_tts_asr.py b/benchmarks/edge_harness/experiments/probe_tts_asr.py new file mode 100644 index 000000000..491c50aa3 --- /dev/null +++ b/benchmarks/edge_harness/experiments/probe_tts_asr.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Transcribe a generated WAV with a separately pinned local ASR checkpoint. + +ASR text agreement is only a proxy for intelligibility, never a speech-quality +or speaker-similarity evaluation. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import time +from pathlib import Path + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _words(text: str) -> list[str]: + return re.findall(r"[a-z0-9]+", text.casefold()) + + +def _word_error_rate(reference: str, actual: str) -> float: + expected, got = _words(reference), _words(actual) + previous = list(range(len(got) + 1)) + for index, token in enumerate(expected, start=1): + current = [index] + for position, candidate in enumerate(got, start=1): + current.append(min( + current[-1] + 1, + previous[position] + 1, + previous[position - 1] + (token != candidate), + )) + previous = current + return previous[-1] / max(len(expected), 1) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wav", type=Path, required=True) + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--model-revision", required=True) + parser.add_argument("--reference-text", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + import librosa + import numpy as np + import soundfile as sf + import torch + import transformers + from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor + + wav, sample_rate = sf.read(args.wav, dtype="float32", always_2d=False) + if wav.ndim != 1 or sample_rate <= 0 or not wav.size or not np.isfinite(wav).all(): + raise ValueError("Expected finite mono audio") + audio_16k = librosa.resample(wav, orig_sr=sample_rate, target_sr=16000) + started = time.perf_counter() + processor = AutoProcessor.from_pretrained(str(args.model_dir), local_files_only=True) + model = AutoModelForSpeechSeq2Seq.from_pretrained( + str(args.model_dir), local_files_only=True + ).to("cpu") + load_s = time.perf_counter() - started + if {parameter.device.type for parameter in model.parameters()} != {"cpu"}: + raise RuntimeError("ASR model was not wholly on CPU") + features = processor( + audio_16k, sampling_rate=16000, return_tensors="pt", + return_attention_mask=True, + ) + started = time.perf_counter() + with torch.no_grad(): + token_ids = model.generate( + features.input_features, attention_mask=features.attention_mask + ) + inference_s = time.perf_counter() - started + transcript = processor.batch_decode(token_ids, skip_special_tokens=True)[0].strip() + report = { + "scope": "local Whisper tiny.en ASR proxy for generated speech; no perceptual quality claim", + "asr_model": "openai/whisper-tiny.en", + "asr_revision": args.model_revision, + "asr_weights_sha256": _sha256(args.model_dir / "model.safetensors"), + "transformers": transformers.__version__, + "torch": torch.__version__, + "wav_sha256": _sha256(args.wav), + "input_sample_rate": sample_rate, + "input_duration_s": wav.size / sample_rate, + "reference_text": args.reference_text, + "transcript": transcript, + "word_error_rate": _word_error_rate(args.reference_text, transcript), + "asr_load_s": load_s, + "asr_inference_s": inference_s, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md index cd881ad8b..33b97e49f 100644 --- a/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md @@ -1,9 +1,10 @@ # 2026-09-23 WSL CPU and native Windows expansion -This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evidence/summary/README.md) with previously unrun WSL CPU and native Windows paths. The September 22 timing tables remain historical. MiniCPM-o now has separate 20-request serial text-to-speech timing runs on WSL CPU and native Windows GPU, plus one synthetic image and one synthetic audio input on each platform; each InternVLA timing remains one observation. Spark has a 20-request Omni output-head component profile on the HX370 AMD NPU and separate 20-request **complete text** profiles through standalone llama.cpp on native Windows CPU and Radeon 890M. None is a sustained or full-modality profile; the standalone Spark paths still need Omni integration. The [checkpoint manifests](../e2e_recovery_20260923/README.md) pin the earlier real weights; the [Spark NPU component record](evidence/spark_amd_npu_output_head/README.md) and [CPU/Radeon whole-model record](evidence/spark_radeon890m_llamacpp/README.md) pin their respective artifacts. No model, precision or context was silently substituted. +This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evidence/summary/README.md) with previously unrun WSL CPU and native Windows paths. The September 22 timing tables remain historical. MiniCPM-o now has separate 20-request serial text-to-speech timing runs on WSL CPU and native Windows GPU, plus one synthetic image and one synthetic audio input on each platform; each InternVLA timing remains one observation. Spark has a 20-request Omni output-head component profile on the HX370 AMD NPU and separate 20-request **complete text** profiles through standalone llama.cpp on native Windows CPU and Radeon 890M. Qwen3-TTS has a separate 20-request complete standalone text-to-WAV profile on native Windows CPU. None is a sustained or full-modality profile; the standalone paths still need Omni integration. The [checkpoint manifests](../e2e_recovery_20260923/README.md) pin the earlier real weights; the [Spark NPU component record](evidence/spark_amd_npu_output_head/README.md), [CPU/Radeon whole-model record](evidence/spark_radeon890m_llamacpp/README.md), and [native CPU Qwen TTS record](evidence/qwen_tts_native_cpu/README.md) pin their respective artifacts. No model, precision or context was silently substituted. | Device / model | Verified depth | Observed output and time | Remaining gate | |---|---|---|---| +| Native Windows HX370 CPU / Qwen3-TTS 0.6B CustomVoice, revision `85e237c12c027371202489a0ec509ded67b5e4b5` | Scoped complete standalone PyTorch text-to-WAV path (P for two named sentences; outside Omni) | Nonzero 24 kHz WAVs of 4.56 s and 3.44 s. For the first prompt, 1 warmup + 20 serial measured complete generations p50/p95 13.94/14.68 s; Whisper tiny.en ASR WER 0.20 on the first and 0.00 on the second sentence. | Omni CPU stage binding and dependency isolation, playable streaming, speech quality, memory loading peak and sustained behavior; [raw evidence](evidence/qwen_tts_native_cpu/README.md). | | Native Windows HX370 CPU / Spark-X2.5-1.7B Q4_K_M GGUF, revision `1f7fa33b1245c14730da39e125714ad3a327901b` | Scoped complete standalone llama.cpp text path (P for two named inputs; outside Omni) | Correct `Paris` and 120-item inventory answer `1511`; 1 warmup + 20 uncached serial full inventory requests p50/p95 6.65/6.86 s, 20/20 correct. Stream-close recovery passed. | Omni whole-model stage binding, broader quality, explicit loading-peak admission and sustained behavior; [raw evidence](evidence/spark_radeon890m_llamacpp/README.md). | | Native Windows HX370 Radeon 890M / Spark-X2.5-1.7B same GGUF | Scoped complete standalone llama.cpp Vulkan1 text path (P for two named inputs; outside Omni) | All 29 logged layer assignments on Vulkan1, with a separate Vulkan_Host buffer; correct `Paris` and `1511`; 1 warmup + 20 uncached serial inventory requests p50/p95 2.18/2.20 s, 20/20 correct. Stream-close recovery passed. | Omni binding, state and memory contract, broad quality, GPU-memory attribution and sustained power/thermal behavior; [raw evidence](evidence/spark_radeon890m_llamacpp/README.md). | | Native Windows HX370 AMD NPU / Spark-X2.5-1.7B output head, revision `448e61eb392c00f2c403185c5b56d5e0665bfaab` | Real-weight RMS norm + full-logit projection component through Omni `external.graph.v1` (B+C, not P) | One fused NPU partition and seven CPU nodes; 4/4 captured activations top-1 match, 55.43–56.81 dB full-logit SNR. One warmup + 20 serial repeated-input component requests: p50/p95 10.21/12.44 ms, versus 17.97/20.29 ms for the original FP32 CPU head through Omni. NPU startup 18.21 s and peak worker RSS 3.52 GiB. [Raw evidence](evidence/spark_amd_npu_output_head/README.md). | Actual vLLM handoff and full generation, transfer cost, quality across inputs/context, memory admission, cancellation and sustained power/thermal behavior. | diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/.gitattributes b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/.gitattributes new file mode 100644 index 000000000..0ee6435fb --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/.gitattributes @@ -0,0 +1,4 @@ +# Preserve native Windows evidence bytes, including CRLF logs and JSON. +*.json -text -eol +*.log -text -eol +*.wav binary diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md new file mode 100644 index 000000000..24ece73a3 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md @@ -0,0 +1,33 @@ +# Native Windows CPU Qwen3-TTS standalone text-to-WAV evidence + +This is a **scoped complete text-to-WAV request outside Omni**, not an Omni streaming or full speech-quality qualification. The CPU-only model placement was asserted on the Ryzen AI 9 HX 370 laptop running Windows 11 build 26200. The installed native vLLM extension still lacks its CPU attention operators, so this experiment used the official Qwen PyTorch wrapper as a candidate whole-model stage backend. + +The local checkpoint is [Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice](https://huggingface.co/Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice/tree/85e237c12c027371202489a0ec509ded67b5e4b5) at revision `85e237c12c027371202489a0ec509ded67b5e4b5`. The talker `model.safetensors` is 1,811,626,576 bytes, SHA256 `bc3c7e785eb961179c25450d1acff03f839e0002f2f3a5aeb67b5735c0fa2adb`; `speech_tokenizer/model.safetensors` is 682,293,092 bytes, SHA256 `836b7b357f5ea43e889936a3709af68dfe3751881acefe4ecf0dbd30ba571258`. Both match the remote revision's LFS hashes. Weights stay in the local model cache, not Git. The [official `qwen-tts` wrapper](https://github.com/QwenLM/Qwen3-TTS) was version 0.1.1. The process used Python 3.12.10, PyTorch 2.13.0+cu130 with CUDA masked, BF16, SDPA, eight CPU threads, `Ryan`/English, seed 42, `max_new_tokens=64`, sampling enabled and `non_streaming_mode=True`. All model parameters were checked on CPU. + +| Run | Input and output | Timing and observed result | Raw record | +|---|---|---|---| +| First sentence | “Hello from the local computer.” → 4.56 s, 24 kHz mono PCM16 | One request took 14.95 s; waveform RMS 0.0610, not silent. Whisper tiny.en transcribed “Hello from the local compute”, WER 0.20. | [report](qwen_tts_cpu_20260923.json), [WAV](qwen_tts_cpu_20260923.wav), [log](qwen_tts_cpu_20260923.log), [ASR report](qwen_tts_cpu_asr_20260923.json), [ASR log](qwen_tts_cpu_asr_20260923.log) | +| Serial profile, same first sentence | 1 warmup excluded, 20 measured requests in one loaded process; each produced the same 4.56 s nonzero waveform | Nearest-rank complete-generation p50 13.94 s, p95 14.68 s; RTF approximately 3.06/3.22. No concurrency or streaming measurement. | [all request timings](qwen_tts_cpu_profile20_20260923.json), [WAV](qwen_tts_cpu_profile20_20260923.wav), [log](qwen_tts_cpu_profile20_20260923.log), [memory samples](qwen_tts_cpu_profile20_memory_samples_20260923.json) | +| Second sentence | “The blue car is parked beside the library.” → 3.44 s, 24 kHz mono PCM16 | One request took 10.55 s; waveform RMS 0.0476. Whisper tiny.en returned the exact sentence, WER 0.00. | [report](qwen_tts_cpu_second_20260923.json), [WAV](qwen_tts_cpu_second_20260923.wav), [log](qwen_tts_cpu_second_20260923.log), [ASR report](qwen_tts_cpu_second_asr_20260923.json), [ASR log](qwen_tts_cpu_second_asr_20260923.log) | + +The memory trace covers only 120 samples over 32.07 s at roughly 0.25 s intervals during the serial profile. Maximum observed Python process working set was 3,202,650,112 bytes and private bytes 5,336,956,928. It is **not** the loading peak, a full-run maximum, or a shared-RAM/iGPU admission measurement. No device power mode or thermal history was captured. The ASR proxy used locally pinned [Whisper tiny.en](https://huggingface.co/openai/whisper-tiny.en/tree/87c7102498dcde7456f24cfd30239ca606ed9063) revision `87c7102498dcde7456f24cfd30239ca606ed9063`, weights SHA256 `db59695928ded6043adaef491a53ef4e12da9611184d77c53baa691a60b958ad`. Neither two transcriptions nor nonzero audio establishes perceptual quality, speaker fidelity, or general intelligibility. + +The native Omni environment has Transformers 5.14.1. Importing `qwen-tts` 0.1.1 with that version [failed](qwen_tts_native_transformers5_import_failure_20260923.log) at `check_model_inputs()`. The successful process overlaid the wrapper's pinned Transformers 4.57.3 and compatible dependencies through an isolated `PYTHONPATH` directory, leaving the Omni environment unchanged. The missing SoX executable emitted a warning; it did not prevent these WAV outputs. This dependency split must be resolved when implementing an Omni stage binding. + +Reproduce in PowerShell after downloading the exact checkpoint and installing `qwen-tts==0.1.1` and its pinned dependencies into isolated local environments: + +```powershell +$root = '\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge' +$env:PYTHONPATH = 'C:\Users\zhout\w2\qwentts_pins' +$env:CUDA_VISIBLE_DEVICES = '' +$env:HF_HUB_OFFLINE = '1' +$env:TRANSFORMERS_OFFLINE = '1' +$env:OMP_NUM_THREADS = '8' +& 'C:\Users\zhout\w2\omni029venv\Scripts\python.exe' "$root\benchmarks\edge_harness\experiments\probe_qwen_tts_native_cpu.py" ` + --model-dir 'C:\Users\zhout\w2\models\Qwen3-TTS-0.6B-85e237c' ` + --output-wav 'C:\Users\zhout\w2\qwen_tts_cpu_profile20_20260923.wav' ` + --output-report 'C:\Users\zhout\w2\qwen_tts_cpu_profile20_20260923.json' ` + --warmups 1 --repeats 20 +``` + +The ASR check is reproduced with [probe_tts_asr.py](../../../../experiments/probe_tts_asr.py), `--wav` pointing to a generated WAV, `--model-dir` pointing to the pinned Whisper cache, `--model-revision 87c7102498dcde7456f24cfd30239ca606ed9063`, and `--reference-text` matching the chosen sentence. This record does not advance the mobile/NPU or combined-device TTS cells. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_20260923.json new file mode 100644 index 000000000..50aef39a6 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_20260923.json @@ -0,0 +1,31 @@ +{ + "scope": "one complete native Windows CPU text-to-WAV request via Qwen's standalone PyTorch wrapper; no Omni streaming or speech-quality claim", + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.12.10", + "torch": "2.13.0+cu130", + "transformers": "4.57.3", + "model_dir": "C:\\Users\\zhout\\w2\\models\\Qwen3-TTS-0.6B-85e237c", + "talker_sha256": "bc3c7e785eb961179c25450d1acff03f839e0002f2f3a5aeb67b5735c0fa2adb", + "speech_tokenizer_sha256": "836b7b357f5ea43e889936a3709af68dfe3751881acefe4ecf0dbd30ba571258", + "model_devices": [ + "cpu" + ], + "dtype": "bfloat16", + "attention": "sdpa", + "threads": 8, + "text": "Hello from the local computer.", + "speaker": "Ryan", + "language": "English", + "max_new_tokens": 64, + "seed": 42, + "startup_s": 2.0423246000427753, + "request_s": 14.94946900010109, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "wav_sha256": "8a41cd36a87981b0ee5b8f77e8a411a4e89b76211fd65b88ad63789f1dd813f4", + "status": "passed" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_20260923.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_20260923.log new file mode 100644 index 000000000..5a19801a2 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_20260923.log @@ -0,0 +1,47 @@ +'sox' is not recognized as an internal or external command, +operable program or batch file. +SoX could not be found! + + If you do not have SoX, proceed here: + - - - http://sox.sourceforge.net/ - - - + + If you do (or think that you should) have SoX, double-check your + path variables. + +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. + +******** +Warning: flash-attn is not installed. Will only run the manual PyTorch version. Please install flash-attn for faster inference. +******** + +{ + "scope": "one complete native Windows CPU text-to-WAV request via Qwen's standalone PyTorch wrapper; no Omni streaming or speech-quality claim", + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.12.10", + "torch": "2.13.0+cu130", + "transformers": "4.57.3", + "model_dir": "C:\\Users\\zhout\\w2\\models\\Qwen3-TTS-0.6B-85e237c", + "talker_sha256": "bc3c7e785eb961179c25450d1acff03f839e0002f2f3a5aeb67b5735c0fa2adb", + "speech_tokenizer_sha256": "836b7b357f5ea43e889936a3709af68dfe3751881acefe4ecf0dbd30ba571258", + "model_devices": [ + "cpu" + ], + "dtype": "bfloat16", + "attention": "sdpa", + "threads": 8, + "text": "Hello from the local computer.", + "speaker": "Ryan", + "language": "English", + "max_new_tokens": 64, + "seed": 42, + "startup_s": 2.0423246000427753, + "request_s": 14.94946900010109, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "wav_sha256": "8a41cd36a87981b0ee5b8f77e8a411a4e89b76211fd65b88ad63789f1dd813f4", + "status": "passed" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_20260923.wav b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_20260923.wav new file mode 100644 index 000000000..d33dea249 Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_20260923.wav differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_asr_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_asr_20260923.json new file mode 100644 index 000000000..ff26fb0da --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_asr_20260923.json @@ -0,0 +1,16 @@ +{ + "scope": "local Whisper tiny.en ASR proxy for generated speech; no perceptual quality claim", + "asr_model": "openai/whisper-tiny.en", + "asr_revision": "87c7102498dcde7456f24cfd30239ca606ed9063", + "asr_weights_sha256": "db59695928ded6043adaef491a53ef4e12da9611184d77c53baa691a60b958ad", + "transformers": "4.57.3", + "torch": "2.13.0+cu130", + "wav_sha256": "8a41cd36a87981b0ee5b8f77e8a411a4e89b76211fd65b88ad63789f1dd813f4", + "input_sample_rate": 24000, + "input_duration_s": 4.56, + "reference_text": "Hello from the local computer.", + "transcript": "Hello from the local compute", + "word_error_rate": 0.2, + "asr_load_s": 0.8311157999560237, + "asr_inference_s": 0.9143147000577301 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_asr_20260923.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_asr_20260923.log new file mode 100644 index 000000000..ed57013f6 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_asr_20260923.log @@ -0,0 +1,17 @@ +Using custom `forced_decoder_ids` from the (generation) config. This is deprecated in favor of the `task` and `language` flags/config options. +{ + "scope": "local Whisper tiny.en ASR proxy for generated speech; no perceptual quality claim", + "asr_model": "openai/whisper-tiny.en", + "asr_revision": "87c7102498dcde7456f24cfd30239ca606ed9063", + "asr_weights_sha256": "db59695928ded6043adaef491a53ef4e12da9611184d77c53baa691a60b958ad", + "transformers": "4.57.3", + "torch": "2.13.0+cu130", + "wav_sha256": "8a41cd36a87981b0ee5b8f77e8a411a4e89b76211fd65b88ad63789f1dd813f4", + "input_sample_rate": 24000, + "input_duration_s": 4.56, + "reference_text": "Hello from the local computer.", + "transcript": "Hello from the local compute", + "word_error_rate": 0.2, + "asr_load_s": 0.8311157999560237, + "asr_inference_s": 0.9143147000577301 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_profile20_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_profile20_20260923.json new file mode 100644 index 000000000..9b84adbb1 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_profile20_20260923.json @@ -0,0 +1,252 @@ +{ + "scope": "one complete native Windows CPU text-to-WAV request via Qwen's standalone PyTorch wrapper; no Omni streaming or speech-quality claim", + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.12.10", + "torch": "2.13.0+cu130", + "transformers": "4.57.3", + "model_dir": "C:\\Users\\zhout\\w2\\models\\Qwen3-TTS-0.6B-85e237c", + "talker_sha256": "bc3c7e785eb961179c25450d1acff03f839e0002f2f3a5aeb67b5735c0fa2adb", + "speech_tokenizer_sha256": "836b7b357f5ea43e889936a3709af68dfe3751881acefe4ecf0dbd30ba571258", + "model_devices": [ + "cpu" + ], + "dtype": "bfloat16", + "attention": "sdpa", + "threads": 8, + "text": "Hello from the local computer.", + "speaker": "Ryan", + "language": "English", + "max_new_tokens": 64, + "seed": 42, + "startup_s": 1.4321526000276208, + "request_s": 14.575387700111605, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "wav_sha256": "8a41cd36a87981b0ee5b8f77e8a411a4e89b76211fd65b88ad63789f1dd813f4", + "warmups": [ + { + "request_s": 14.520753899938427, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + } + ], + "measured": [ + { + "request_s": 14.575387700111605, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 14.68431669997517, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.939570999937132, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.917316399980336, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.882408199948259, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.867731199949048, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.773739999975078, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.850018199998885, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.97315980005078, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 14.145962399896234, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.959658100036904, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.83213820005767, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 14.004250700003468, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.633021700079553, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 14.114254100015387, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.691989900078624, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.85189100005664, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 14.763823200017214, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.973342600045726, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 14.051946000079624, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + } + ], + "profile": { + "repeats": 20, + "concurrency": 1, + "p50_request_s": 13.939570999937132, + "p95_request_s": 14.68431669997517, + "all_same_float32_audio": true + }, + "status": "passed" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_profile20_20260923.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_profile20_20260923.log new file mode 100644 index 000000000..6b8fa4198 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_profile20_20260923.log @@ -0,0 +1,288 @@ +'sox' is not recognized as an internal or external command, +operable program or batch file. +SoX could not be found! + + If you do not have SoX, proceed here: + - - - http://sox.sourceforge.net/ - - - + + If you do (or think that you should) have SoX, double-check your + path variables. + +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. + +******** +Warning: flash-attn is not installed. Will only run the manual PyTorch version. Please install flash-attn for faster inference. +******** + +{ + "scope": "one complete native Windows CPU text-to-WAV request via Qwen's standalone PyTorch wrapper; no Omni streaming or speech-quality claim", + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.12.10", + "torch": "2.13.0+cu130", + "transformers": "4.57.3", + "model_dir": "C:\\Users\\zhout\\w2\\models\\Qwen3-TTS-0.6B-85e237c", + "talker_sha256": "bc3c7e785eb961179c25450d1acff03f839e0002f2f3a5aeb67b5735c0fa2adb", + "speech_tokenizer_sha256": "836b7b357f5ea43e889936a3709af68dfe3751881acefe4ecf0dbd30ba571258", + "model_devices": [ + "cpu" + ], + "dtype": "bfloat16", + "attention": "sdpa", + "threads": 8, + "text": "Hello from the local computer.", + "speaker": "Ryan", + "language": "English", + "max_new_tokens": 64, + "seed": 42, + "startup_s": 1.4321526000276208, + "request_s": 14.575387700111605, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "wav_sha256": "8a41cd36a87981b0ee5b8f77e8a411a4e89b76211fd65b88ad63789f1dd813f4", + "warmups": [ + { + "request_s": 14.520753899938427, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + } + ], + "measured": [ + { + "request_s": 14.575387700111605, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 14.68431669997517, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.939570999937132, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.917316399980336, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.882408199948259, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.867731199949048, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.773739999975078, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.850018199998885, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.97315980005078, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 14.145962399896234, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.959658100036904, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.83213820005767, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 14.004250700003468, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.633021700079553, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 14.114254100015387, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.691989900078624, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.85189100005664, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 14.763823200017214, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 13.973342600045726, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + }, + { + "request_s": 14.051946000079624, + "sample_rate": 24000, + "samples": 109440, + "duration_s": 4.56, + "rms": 0.060994377256646456, + "peak_abs": 0.38671875, + "nonzero_samples": 109440, + "float32_sha256": "ba2a318b6f62c24dd94194f959676e93c9e739944b2b652cc9e250fadd547d96" + } + ], + "profile": { + "repeats": 20, + "concurrency": 1, + "p50_request_s": 13.939570999937132, + "p95_request_s": 14.68431669997517, + "all_same_float32_audio": true + }, + "status": "passed" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_profile20_20260923.wav b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_profile20_20260923.wav new file mode 100644 index 000000000..d33dea249 Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_profile20_20260923.wav differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_profile20_memory_samples_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_profile20_memory_samples_20260923.json new file mode 100644 index 000000000..5ee3ab963 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_profile20_memory_samples_20260923.json @@ -0,0 +1,602 @@ +[ + { + "unix": 1790161689.088, + "working_set_bytes": 3060117504, + "private_bytes": 4949221376 + }, + { + "unix": 1790161689.386, + "working_set_bytes": 3202650112, + "private_bytes": 5313794048 + }, + { + "unix": 1790161689.652, + "working_set_bytes": 3020541952, + "private_bytes": 4763381760 + }, + { + "unix": 1790161689.916, + "working_set_bytes": 3022069760, + "private_bytes": 4765212672 + }, + { + "unix": 1790161690.182, + "working_set_bytes": 3021770752, + "private_bytes": 4764819456 + }, + { + "unix": 1790161690.451, + "working_set_bytes": 3022278656, + "private_bytes": 4764884992 + }, + { + "unix": 1790161690.72, + "working_set_bytes": 3023749120, + "private_bytes": 4767051776 + }, + { + "unix": 1790161690.989, + "working_set_bytes": 3024019456, + "private_bytes": 4767117312 + }, + { + "unix": 1790161691.257, + "working_set_bytes": 3023998976, + "private_bytes": 4767117312 + }, + { + "unix": 1790161691.524, + "working_set_bytes": 3025113088, + "private_bytes": 4767838208 + }, + { + "unix": 1790161691.793, + "working_set_bytes": 3025469440, + "private_bytes": 4768428032 + }, + { + "unix": 1790161692.062, + "working_set_bytes": 3025842176, + "private_bytes": 4768952320 + }, + { + "unix": 1790161692.331, + "working_set_bytes": 3025772544, + "private_bytes": 4768493568 + }, + { + "unix": 1790161692.6, + "working_set_bytes": 3025969152, + "private_bytes": 4768624640 + }, + { + "unix": 1790161692.871, + "working_set_bytes": 3026030592, + "private_bytes": 4768624640 + }, + { + "unix": 1790161693.14, + "working_set_bytes": 3026505728, + "private_bytes": 4769083392 + }, + { + "unix": 1790161693.41, + "working_set_bytes": 3030134784, + "private_bytes": 4772892672 + }, + { + "unix": 1790161693.695, + "working_set_bytes": 3030298624, + "private_bytes": 4773023744 + }, + { + "unix": 1790161693.965, + "working_set_bytes": 3030384640, + "private_bytes": 4772958208 + }, + { + "unix": 1790161694.231, + "working_set_bytes": 3030425600, + "private_bytes": 4773023744 + }, + { + "unix": 1790161694.498, + "working_set_bytes": 3030360064, + "private_bytes": 4772958208 + }, + { + "unix": 1790161694.767, + "working_set_bytes": 3030302720, + "private_bytes": 4772958208 + }, + { + "unix": 1790161695.033, + "working_set_bytes": 3030401024, + "private_bytes": 4773023744 + }, + { + "unix": 1790161695.304, + "working_set_bytes": 3030454272, + "private_bytes": 4773023744 + }, + { + "unix": 1790161695.573, + "working_set_bytes": 3030921216, + "private_bytes": 4773482496 + }, + { + "unix": 1790161695.854, + "working_set_bytes": 3030863872, + "private_bytes": 4773482496 + }, + { + "unix": 1790161696.124, + "working_set_bytes": 3031007232, + "private_bytes": 4773613568 + }, + { + "unix": 1790161696.392, + "working_set_bytes": 3031044096, + "private_bytes": 4773613568 + }, + { + "unix": 1790161696.657, + "working_set_bytes": 3031052288, + "private_bytes": 4773613568 + }, + { + "unix": 1790161696.927, + "working_set_bytes": 3030994944, + "private_bytes": 4773548032 + }, + { + "unix": 1790161697.196, + "working_set_bytes": 3031031808, + "private_bytes": 4773613568 + }, + { + "unix": 1790161697.464, + "working_set_bytes": 3031068672, + "private_bytes": 4773613568 + }, + { + "unix": 1790161697.732, + "working_set_bytes": 3031093248, + "private_bytes": 4773613568 + }, + { + "unix": 1790161698, + "working_set_bytes": 3031076864, + "private_bytes": 4773613568 + }, + { + "unix": 1790161698.271, + "working_set_bytes": 3031601152, + "private_bytes": 4774141952 + }, + { + "unix": 1790161698.541, + "working_set_bytes": 3031568384, + "private_bytes": 4774141952 + }, + { + "unix": 1790161698.808, + "working_set_bytes": 3031793664, + "private_bytes": 4774338560 + }, + { + "unix": 1790161699.077, + "working_set_bytes": 3031814144, + "private_bytes": 4774338560 + }, + { + "unix": 1790161699.347, + "working_set_bytes": 3031814144, + "private_bytes": 4774338560 + }, + { + "unix": 1790161699.617, + "working_set_bytes": 3031838720, + "private_bytes": 4774338560 + }, + { + "unix": 1790161699.887, + "working_set_bytes": 3032080384, + "private_bytes": 4774535168 + }, + { + "unix": 1790161700.153, + "working_set_bytes": 3032080384, + "private_bytes": 4774535168 + }, + { + "unix": 1790161700.422, + "working_set_bytes": 3032616960, + "private_bytes": 4775059456 + }, + { + "unix": 1790161700.691, + "working_set_bytes": 3036090368, + "private_bytes": 4778541056 + }, + { + "unix": 1790161700.96, + "working_set_bytes": 3036614656, + "private_bytes": 4779065344 + }, + { + "unix": 1790161701.226, + "working_set_bytes": 3036688384, + "private_bytes": 4779196416 + }, + { + "unix": 1790161701.508, + "working_set_bytes": 3036688384, + "private_bytes": 4779196416 + }, + { + "unix": 1790161701.787, + "working_set_bytes": 3036684288, + "private_bytes": 4779196416 + }, + { + "unix": 1790161702.055, + "working_set_bytes": 3036700672, + "private_bytes": 4779196416 + }, + { + "unix": 1790161702.324, + "working_set_bytes": 3036766208, + "private_bytes": 4779261952 + }, + { + "unix": 1790161702.593, + "working_set_bytes": 3036786688, + "private_bytes": 4779327488 + }, + { + "unix": 1790161702.861, + "working_set_bytes": 3037073408, + "private_bytes": 4796727296 + }, + { + "unix": 1790161703.13, + "working_set_bytes": 3129810944, + "private_bytes": 4941004800 + }, + { + "unix": 1790161703.396, + "working_set_bytes": 3134775296, + "private_bytes": 4940738560 + }, + { + "unix": 1790161703.665, + "working_set_bytes": 3020894208, + "private_bytes": 4795482112 + }, + { + "unix": 1790161703.936, + "working_set_bytes": 3019825152, + "private_bytes": 4795088896 + }, + { + "unix": 1790161704.204, + "working_set_bytes": 3020812288, + "private_bytes": 4796137472 + }, + { + "unix": 1790161704.472, + "working_set_bytes": 3023216640, + "private_bytes": 4795744256 + }, + { + "unix": 1790161704.741, + "working_set_bytes": 3023912960, + "private_bytes": 4796334080 + }, + { + "unix": 1790161705.01, + "working_set_bytes": 3024146432, + "private_bytes": 4796530688 + }, + { + "unix": 1790161705.277, + "working_set_bytes": 3024134144, + "private_bytes": 4796530688 + }, + { + "unix": 1790161705.544, + "working_set_bytes": 3024945152, + "private_bytes": 4796465152 + }, + { + "unix": 1790161705.815, + "working_set_bytes": 3025084416, + "private_bytes": 4796596224 + }, + { + "unix": 1790161706.084, + "working_set_bytes": 3026300928, + "private_bytes": 4796727296 + }, + { + "unix": 1790161706.351, + "working_set_bytes": 3026055168, + "private_bytes": 4796399616 + }, + { + "unix": 1790161706.622, + "working_set_bytes": 3026067456, + "private_bytes": 4796399616 + }, + { + "unix": 1790161706.888, + "working_set_bytes": 3026100224, + "private_bytes": 4796399616 + }, + { + "unix": 1790161707.156, + "working_set_bytes": 3028156416, + "private_bytes": 4796661760 + }, + { + "unix": 1790161707.425, + "working_set_bytes": 3029786624, + "private_bytes": 4796596224 + }, + { + "unix": 1790161707.694, + "working_set_bytes": 3030142976, + "private_bytes": 4796727296 + }, + { + "unix": 1790161707.966, + "working_set_bytes": 3030380544, + "private_bytes": 4796596224 + }, + { + "unix": 1790161708.235, + "working_set_bytes": 3030433792, + "private_bytes": 4796661760 + }, + { + "unix": 1790161708.504, + "working_set_bytes": 3030487040, + "private_bytes": 4796727296 + }, + { + "unix": 1790161708.772, + "working_set_bytes": 3030441984, + "private_bytes": 4796727296 + }, + { + "unix": 1790161709.042, + "working_set_bytes": 3030859776, + "private_bytes": 4796727296 + }, + { + "unix": 1790161709.312, + "working_set_bytes": 3030872064, + "private_bytes": 4796727296 + }, + { + "unix": 1790161709.58, + "working_set_bytes": 3030880256, + "private_bytes": 4796727296 + }, + { + "unix": 1790161709.848, + "working_set_bytes": 3030880256, + "private_bytes": 4796727296 + }, + { + "unix": 1790161710.119, + "working_set_bytes": 3030872064, + "private_bytes": 4796727296 + }, + { + "unix": 1790161710.4, + "working_set_bytes": 3030933504, + "private_bytes": 4796727296 + }, + { + "unix": 1790161710.667, + "working_set_bytes": 3030974464, + "private_bytes": 4796727296 + }, + { + "unix": 1790161710.935, + "working_set_bytes": 3030949888, + "private_bytes": 4796727296 + }, + { + "unix": 1790161711.202, + "working_set_bytes": 3030884352, + "private_bytes": 4796661760 + }, + { + "unix": 1790161711.482, + "working_set_bytes": 3030958080, + "private_bytes": 4796727296 + }, + { + "unix": 1790161711.751, + "working_set_bytes": 3031097344, + "private_bytes": 4796727296 + }, + { + "unix": 1790161712.02, + "working_set_bytes": 3031556096, + "private_bytes": 4796661760 + }, + { + "unix": 1790161712.287, + "working_set_bytes": 3031564288, + "private_bytes": 4796661760 + }, + { + "unix": 1790161712.556, + "working_set_bytes": 3031564288, + "private_bytes": 4796661760 + }, + { + "unix": 1790161712.825, + "working_set_bytes": 3032092672, + "private_bytes": 4796727296 + }, + { + "unix": 1790161713.091, + "working_set_bytes": 3032088576, + "private_bytes": 4796727296 + }, + { + "unix": 1790161713.355, + "working_set_bytes": 3032088576, + "private_bytes": 4796727296 + }, + { + "unix": 1790161713.636, + "working_set_bytes": 3032027136, + "private_bytes": 4796596224 + }, + { + "unix": 1790161713.906, + "working_set_bytes": 3032174592, + "private_bytes": 4796727296 + }, + { + "unix": 1790161714.175, + "working_set_bytes": 3032186880, + "private_bytes": 4796727296 + }, + { + "unix": 1790161714.441, + "working_set_bytes": 3036643328, + "private_bytes": 4796596224 + }, + { + "unix": 1790161714.712, + "working_set_bytes": 3036839936, + "private_bytes": 4796596224 + }, + { + "unix": 1790161714.98, + "working_set_bytes": 3036803072, + "private_bytes": 4796727296 + }, + { + "unix": 1790161715.249, + "working_set_bytes": 3036721152, + "private_bytes": 4796596224 + }, + { + "unix": 1790161715.519, + "working_set_bytes": 3036811264, + "private_bytes": 4796727296 + }, + { + "unix": 1790161715.788, + "working_set_bytes": 3036815360, + "private_bytes": 4796727296 + }, + { + "unix": 1790161716.056, + "working_set_bytes": 3036819456, + "private_bytes": 4796727296 + }, + { + "unix": 1790161716.325, + "working_set_bytes": 3036643328, + "private_bytes": 4796399616 + }, + { + "unix": 1790161716.594, + "working_set_bytes": 3036815360, + "private_bytes": 4796661760 + }, + { + "unix": 1790161716.863, + "working_set_bytes": 3062509568, + "private_bytes": 4818477056 + }, + { + "unix": 1790161717.128, + "working_set_bytes": 3131551744, + "private_bytes": 5092241408 + }, + { + "unix": 1790161717.397, + "working_set_bytes": 3198607360, + "private_bytes": 5336956928 + }, + { + "unix": 1790161717.668, + "working_set_bytes": 3021910016, + "private_bytes": 4765679616 + }, + { + "unix": 1790161717.937, + "working_set_bytes": 3021344768, + "private_bytes": 4764565504 + }, + { + "unix": 1790161718.207, + "working_set_bytes": 3021414400, + "private_bytes": 4764434432 + }, + { + "unix": 1790161718.478, + "working_set_bytes": 3023622144, + "private_bytes": 4766404608 + }, + { + "unix": 1790161718.748, + "working_set_bytes": 3024072704, + "private_bytes": 4767584256 + }, + { + "unix": 1790161719.012, + "working_set_bytes": 3024179200, + "private_bytes": 4767256576 + }, + { + "unix": 1790161719.282, + "working_set_bytes": 3024273408, + "private_bytes": 4767846400 + }, + { + "unix": 1790161719.547, + "working_set_bytes": 3024846848, + "private_bytes": 4767780864 + }, + { + "unix": 1790161719.817, + "working_set_bytes": 3024887808, + "private_bytes": 4767846400 + }, + { + "unix": 1790161720.086, + "working_set_bytes": 3026202624, + "private_bytes": 4768960512 + }, + { + "unix": 1790161720.356, + "working_set_bytes": 3025444864, + "private_bytes": 4768370688 + }, + { + "unix": 1790161720.622, + "working_set_bytes": 3025362944, + "private_bytes": 4768829440 + }, + { + "unix": 1790161720.892, + "working_set_bytes": 3025711104, + "private_bytes": 4768829440 + }, + { + "unix": 1790161721.16, + "working_set_bytes": 3029348352, + "private_bytes": 4773421056 + } +] diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_20260923.json new file mode 100644 index 000000000..373e5bae6 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_20260923.json @@ -0,0 +1,51 @@ +{ + "scope": "one complete native Windows CPU text-to-WAV request via Qwen's standalone PyTorch wrapper; no Omni streaming or speech-quality claim", + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.12.10", + "torch": "2.13.0+cu130", + "transformers": "4.57.3", + "model_dir": "C:\\Users\\zhout\\w2\\models\\Qwen3-TTS-0.6B-85e237c", + "talker_sha256": "bc3c7e785eb961179c25450d1acff03f839e0002f2f3a5aeb67b5735c0fa2adb", + "speech_tokenizer_sha256": "836b7b357f5ea43e889936a3709af68dfe3751881acefe4ecf0dbd30ba571258", + "model_devices": [ + "cpu" + ], + "dtype": "bfloat16", + "attention": "sdpa", + "threads": 8, + "text": "The blue car is parked beside the library.", + "speaker": "Ryan", + "language": "English", + "max_new_tokens": 64, + "seed": 42, + "startup_s": 1.4111617000307888, + "request_s": 10.552245300030336, + "sample_rate": 24000, + "samples": 82560, + "duration_s": 3.44, + "rms": 0.04756783349280258, + "peak_abs": 0.32421875, + "nonzero_samples": 82560, + "wav_sha256": "2e93e1ef8858df3a263fcdbbbe2b9bed8376485042dbc640092967cdb6afca2b", + "warmups": [], + "measured": [ + { + "request_s": 10.552245300030336, + "sample_rate": 24000, + "samples": 82560, + "duration_s": 3.44, + "rms": 0.04756783349280258, + "peak_abs": 0.32421875, + "nonzero_samples": 82560, + "float32_sha256": "97195646a0fe6b42627040f3c60181b7601fe6de5d524dd57219a69b9abb66c2" + } + ], + "profile": { + "repeats": 1, + "concurrency": 1, + "p50_request_s": 10.552245300030336, + "p95_request_s": 10.552245300030336, + "all_same_float32_audio": true + }, + "status": "passed" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_20260923.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_20260923.log new file mode 100644 index 000000000..398046b0f --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_20260923.log @@ -0,0 +1,67 @@ +'sox' is not recognized as an internal or external command, +operable program or batch file. +SoX could not be found! + + If you do not have SoX, proceed here: + - - - http://sox.sourceforge.net/ - - - + + If you do (or think that you should) have SoX, double-check your + path variables. + +Setting `pad_token_id` to `eos_token_id`:2150 for open-end generation. + +******** +Warning: flash-attn is not installed. Will only run the manual PyTorch version. Please install flash-attn for faster inference. +******** + +{ + "scope": "one complete native Windows CPU text-to-WAV request via Qwen's standalone PyTorch wrapper; no Omni streaming or speech-quality claim", + "platform": "Windows-11-10.0.26200-SP0", + "python": "3.12.10", + "torch": "2.13.0+cu130", + "transformers": "4.57.3", + "model_dir": "C:\\Users\\zhout\\w2\\models\\Qwen3-TTS-0.6B-85e237c", + "talker_sha256": "bc3c7e785eb961179c25450d1acff03f839e0002f2f3a5aeb67b5735c0fa2adb", + "speech_tokenizer_sha256": "836b7b357f5ea43e889936a3709af68dfe3751881acefe4ecf0dbd30ba571258", + "model_devices": [ + "cpu" + ], + "dtype": "bfloat16", + "attention": "sdpa", + "threads": 8, + "text": "The blue car is parked beside the library.", + "speaker": "Ryan", + "language": "English", + "max_new_tokens": 64, + "seed": 42, + "startup_s": 1.4111617000307888, + "request_s": 10.552245300030336, + "sample_rate": 24000, + "samples": 82560, + "duration_s": 3.44, + "rms": 0.04756783349280258, + "peak_abs": 0.32421875, + "nonzero_samples": 82560, + "wav_sha256": "2e93e1ef8858df3a263fcdbbbe2b9bed8376485042dbc640092967cdb6afca2b", + "warmups": [], + "measured": [ + { + "request_s": 10.552245300030336, + "sample_rate": 24000, + "samples": 82560, + "duration_s": 3.44, + "rms": 0.04756783349280258, + "peak_abs": 0.32421875, + "nonzero_samples": 82560, + "float32_sha256": "97195646a0fe6b42627040f3c60181b7601fe6de5d524dd57219a69b9abb66c2" + } + ], + "profile": { + "repeats": 1, + "concurrency": 1, + "p50_request_s": 10.552245300030336, + "p95_request_s": 10.552245300030336, + "all_same_float32_audio": true + }, + "status": "passed" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_20260923.wav b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_20260923.wav new file mode 100644 index 000000000..5dd98bb6b Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_20260923.wav differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_asr_20260923.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_asr_20260923.json new file mode 100644 index 000000000..5b7c3466c --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_asr_20260923.json @@ -0,0 +1,16 @@ +{ + "scope": "local Whisper tiny.en ASR proxy for generated speech; no perceptual quality claim", + "asr_model": "openai/whisper-tiny.en", + "asr_revision": "87c7102498dcde7456f24cfd30239ca606ed9063", + "asr_weights_sha256": "db59695928ded6043adaef491a53ef4e12da9611184d77c53baa691a60b958ad", + "transformers": "4.57.3", + "torch": "2.13.0+cu130", + "wav_sha256": "2e93e1ef8858df3a263fcdbbbe2b9bed8376485042dbc640092967cdb6afca2b", + "input_sample_rate": 24000, + "input_duration_s": 3.44, + "reference_text": "The blue car is parked beside the library.", + "transcript": "The blue car is parked beside the library.", + "word_error_rate": 0.0, + "asr_load_s": 0.649280299898237, + "asr_inference_s": 0.9257399999769405 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_asr_20260923.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_asr_20260923.log new file mode 100644 index 000000000..2109f06d8 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_cpu_second_asr_20260923.log @@ -0,0 +1,17 @@ +Using custom `forced_decoder_ids` from the (generation) config. This is deprecated in favor of the `task` and `language` flags/config options. +{ + "scope": "local Whisper tiny.en ASR proxy for generated speech; no perceptual quality claim", + "asr_model": "openai/whisper-tiny.en", + "asr_revision": "87c7102498dcde7456f24cfd30239ca606ed9063", + "asr_weights_sha256": "db59695928ded6043adaef491a53ef4e12da9611184d77c53baa691a60b958ad", + "transformers": "4.57.3", + "torch": "2.13.0+cu130", + "wav_sha256": "2e93e1ef8858df3a263fcdbbbe2b9bed8376485042dbc640092967cdb6afca2b", + "input_sample_rate": 24000, + "input_duration_s": 3.44, + "reference_text": "The blue car is parked beside the library.", + "transcript": "The blue car is parked beside the library.", + "word_error_rate": 0.0, + "asr_load_s": 0.649280299898237, + "asr_inference_s": 0.9257399999769405 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_native_transformers5_import_failure_20260923.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_native_transformers5_import_failure_20260923.log new file mode 100644 index 000000000..91d82abad --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/qwen_tts_native_transformers5_import_failure_20260923.log @@ -0,0 +1,29 @@ +'sox' is not recognized as an internal or external command, +operable program or batch file. +SoX could not be found! + + If you do not have SoX, proceed here: + - - - http://sox.sourceforge.net/ - - - + + If you do (or think that you should) have SoX, double-check your + path variables. + +Traceback (most recent call last): + File "", line 1, in + File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\qwen_tts\__init__.py", line 21, in + from .inference.qwen3_tts_model import Qwen3TTSModel, VoiceClonePromptItem + File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\qwen_tts\inference\qwen3_tts_model.py", line 29, in + from ..core.models import Qwen3TTSConfig, Qwen3TTSForConditionalGeneration, Qwen3TTSProcessor + File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\qwen_tts\core\__init__.py", line 19, in + from .tokenizer_12hz.modeling_qwen3_tts_tokenizer_v2 import Qwen3TTSTokenizerV2Model + File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\qwen_tts\core\tokenizer_12hz\modeling_qwen3_tts_tokenizer_v2.py", line 476, in + class Qwen3TTSTokenizerV2DecoderTransformerModel(Qwen3TTSTokenizerV2DecoderPreTrainedModel): + File "C:\Users\zhout\w2\omni029venv\Lib\site-packages\qwen_tts\core\tokenizer_12hz\modeling_qwen3_tts_tokenizer_v2.py", line 499, in Qwen3TTSTokenizerV2DecoderTransformerModel + @check_model_inputs() + ^^^^^^^^^^^^^^^^^^^^ +TypeError: check_model_inputs() missing 1 required positional argument: 'func' + +******** +Warning: flash-attn is not installed. Will only run the manual PyTorch version. Please install flash-attn for faster inference. +******** + diff --git a/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md b/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md index 3d824a6f4..86ebed80f 100644 --- a/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md +++ b/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md @@ -1,6 +1,6 @@ # E2E check and profiling -All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [WSL CPU/native Windows expansion](../../../e2e_expansion_20260923/README.md). The expansion adds separate 20-request serial MiniCPM-o text-to-speech profiles on WSL CPU and native Windows GPU, a Spark output-head component profile on the HX370 AMD NPU, and complete Spark text-request profiles through standalone llama.cpp on native Windows CPU and Radeon 890M. Profiling completion does not imply full release qualification or Omni integration. +All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [WSL CPU/native Windows expansion](../../../e2e_expansion_20260923/README.md). The expansion adds separate 20-request serial MiniCPM-o text-to-speech profiles on WSL CPU and native Windows GPU, a Spark output-head component profile on the HX370 AMD NPU, complete Spark text-request profiles through standalone llama.cpp on native Windows CPU and Radeon 890M, and a standalone Qwen3-TTS text-to-WAV profile on native Windows CPU. Profiling completion does not imply full release qualification or Omni integration. ## Configuration × model @@ -9,7 +9,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i | Device | Spark-X2.5 | Qwen3-TTS 0.6B CustomVoice | Qwen3.8-27B | MiniCPM-o 4.5 | InternVLA-A1 | |---|---|---|---|---|---| | PC HX370: CPU / WSL | profile completed | scoped E2E: short stream | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass | -| PC HX370: CPU / native Windows | scoped E2E: standalone Spark text | NOT E2E | NOT E2E | NOT E2E | synthetic policy pass | +| PC HX370: CPU / native Windows | scoped E2E: standalone Spark text | scoped E2E: standalone text → WAV | NOT E2E | NOT E2E | synthetic policy pass | | PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text/image/audio → speech | synthetic policy pass | | PC HX370 + RTX 5090 Laptop / Windows | profile completed | profile completed | NOT E2E | scoped E2E: text/image/audio → speech | synthetic policy pass | | PC HX370 + Radeon 890M / Windows worker | scoped E2E: standalone Spark text | NOT E2E | NOT E2E | NOT E2E | NOT E2E | @@ -31,7 +31,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i | PC HX370: CPU / WSL | MiniCPM-o 4.5 | SCOPED_E2E | The verified checkpoint completed real three-stage text, one synthetic image and one synthetic audio input to text+speech on the vLLM 0.28 CPU backend in separate bounded plans. The 448×448 red square was described correctly with a nonzero 5.2 s WAV in one 37.70 s request; the 2 s/440 Hz tone was described as “A loud and high pitched beep” with a nonzero 2.16 s WAV in one 17.30 s request. CPU wording/token IDs differ from the WSL GPU runs, with no task-quality threshold established. A separate 1-warmup/20-measured serial **text** run had request wall p50 21.86 s/p95 22.97 s. WSL swap reached 5.75 GB during that text profile; manual snapshots in the multimodal runs showed heavy swap use but do not prove peak memory. Both multimodal runs force-killed stage 0 at shutdown and reported shared-memory cleanup warnings. | Validate real image/audio suites, speech intelligibility, video input, concurrency, interruption, admission/loading peak, clean shutdown and sustained thermal behavior before broader CPU qualification; [CPU evidence](../../../e2e_expansion_20260923/README.md). | | PC HX370: CPU / WSL | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded in CPU-only PyTorch and produced finite `[1, 50, 32]` actions from zero observations/noise in one 3.25 s forward. Compared with the same CUDA checkpoint/input, relative L2 error was 1.30% and cosine similarity 0.999917; no task-quality tolerance is established. | Validate real observations against reference actions, units/timestamps, repeated latency and memory before robot-task qualification; [CPU evidence](../../../e2e_expansion_20260923/README.md). | | PC HX370: CPU / native Windows | Spark-X2.5 | SCOPED_E2E_STANDALONE | The real Spark-X2.5-1.7B Q4_K_M GGUF ran a complete text path in native Windows llama.cpp CPU (`-dev none -ngl 0`): `Paris` and the correct `1511` answer to a 2,463-token inventory prompt, one warmup plus 20 uncached serial inventory requests with p50/p95 6.65/6.86 s and 20/20 correct, and a post-stream-close recovery request. This does not repair the missing vLLM CPU attention operators or establish an Omni stage binding. One repeated input, no loading peak or sustained thermal/power gate. | Bind the whole llama.cpp session to Omni's state, request/event, memory and cancellation contracts; validate broader text quality and sustained use; [standalone backend evidence](../../../e2e_expansion_20260923/evidence/spark_radeon890m_llamacpp/README.md). | -| PC HX370: CPU / native Windows | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | The installed native Windows vLLM extension lacks cpu_attn_reshape_and_cache and cpu_attn_get_scheduler_metadata. The separate ORT CPU graph worker is not a whole-model text/TTS adapter; a qualified CPU build/binding is missing. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | +| PC HX370: CPU / native Windows | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E_STANDALONE | The pinned 0.6B CustomVoice checkpoint ran wholly on CPU through Qwen's standalone PyTorch wrapper (BF16/SDPA) and produced nonzero 24 kHz WAVs for two English sentences. For one repeated short prompt, 1 warmup + 20 serial measured complete generation requests had nearest-rank p50/p95 13.94/14.68 s for 4.56 s of audio. Independent Whisper tiny.en ASR gave word error rates 0.20 and 0.00 on the two WAVs; this is only an intelligibility proxy. The native vLLM CPU extension still lacks the required attention operators. No Omni binding, playable streaming, listening/quality threshold, loading peak or sustained power/thermal run was verified. | Bind the whole CPU TTS backend to Omni's stage and streaming/state contracts; validate playback, quality, cancellation, admission and sustained use; [standalone TTS evidence](../../../e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md). | | PC HX370: CPU / native Windows | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The installed native Windows vLLM extension lacks cpu_attn_reshape_and_cache and cpu_attn_get_scheduler_metadata. The separate ORT CPU graph worker is not a whole-model text/TTS adapter; a qualified CPU build/binding is missing. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | PC HX370: CPU / native Windows | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | The exact checkpoint is verified in the local Linux model store, but the installed native Windows vLLM extension lacks required CPU attention operators; no native full-model run or memory admission exists. | Establish a native Windows CPU backend and model access, then run the full requested path; [weight evidence](../../../e2e_recovery_20260923/evidence/weights/minicpmo.json). | | PC HX370: CPU / native Windows | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded from a UNC path and the direct CPU policy produced finite `[1, 50, 32]` actions on synthetic zero observations/noise. Policy/output CPU placement was checked; one forward took 3.36 s. Relative L2 difference from WSL CPU was 1.08% (no task tolerance). This direct policy route does not require native vLLM CPU attention operators. | Validate real observations/reference actions, units/timestamps, memory and repeated latency before task qualification; [native evidence](../../../e2e_expansion_20260923/README.md). | diff --git a/docs/design/omni_edge_e2e_roadmap.md b/docs/design/omni_edge_e2e_roadmap.md index f7bdb0536..111ad62f3 100644 --- a/docs/design/omni_edge_e2e_roadmap.md +++ b/docs/design/omni_edge_e2e_roadmap.md @@ -46,6 +46,16 @@ from GPU, swap is heavily used, and stage 0 requires forced shutdown with shared-memory cleanup warnings. These single requests do not inherit the 20-request text latency profile or establish real image/audio quality. +Native Windows HX370 CPU now also completes two Qwen3-TTS CustomVoice +text-to-WAV requests through Qwen's standalone PyTorch wrapper using the pinned +0.6B checkpoint. A separate 1-warmup/20-measured serial profile of one short +prompt took p50/p95 13.94/14.68 s to generate 4.56 s of audio. Whisper tiny.en +ASR gave word error rates 0.20 and 0.00 for the two WAVs, an intelligibility +proxy rather than a quality pass. The [raw record](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/qwen_tts_native_cpu/README.md) +documents the isolated Transformers 4.57.3 dependency and short process-memory +trace. Omni CPU stage binding, playable streaming, loading-peak admission and +sustained power/thermal tests remain M2 work. + ## Architecture and prerequisites Keep Omni's PipelineConfig, StageRuntime, StageClient, orchestration and admission