Audit X Elite CPU vocoder and isolate TFLite conversion parity
This commit is contained in:
tzhouam committed
1 parent
e4a4477929
commit
ea373e3cbf
11 files changed
+5025
-6
No files matched your search
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit one hosted Qwen3-TTS talker decode step against pinned ONNX CPU tensors."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
SOURCE_SHA256 = "41feb08517678e94d9301e992d0df610fc60e7c9faba5600703298c86554fd22"
|
||||
FIXTURE_SHA256 = "0bf538e1b64dfc9737dc4f925d61c6e4ffa5075a5dcd7afb1e85ac3bd2d9afea"
|
||||
REFERENCE_SHA256 = "ee0ead70e33c02c7d3d6bcb757c5cf0fcbb9eb6fc75c34abfbe8a4479657d5be"
|
||||
COMPILE_OPTIONS = "--target_runtime qnn_context_binary --quantize_full_type float16"
|
||||
INPUT_NAMES = ["x", "cos", "sin", "mask"] + [
|
||||
name for layer in range(28) for name in (f"k_cache_{layer}", f"v_cache_{layer}")
|
||||
]
|
||||
OUTPUT_NAMES = ["logits"] + [
|
||||
name for layer in range(28) for name in (f"k_new_{layer}", f"v_new_{layer}")
|
||||
]
|
||||
|
||||
|
||||
def file_record(path: Path) -> dict:
|
||||
with path.open("rb") as source:
|
||||
digest = hashlib.file_digest(source, "sha256").hexdigest()
|
||||
return {"bytes": path.stat().st_size, "sha256": digest}
|
||||
|
||||
|
||||
def require_array(data: np.lib.npyio.NpzFile, name: str, shape: tuple[int, ...]) -> np.ndarray:
|
||||
value = np.asarray(data[name])
|
||||
if value.shape != shape or value.dtype != np.float32 or not np.isfinite(value).all():
|
||||
raise ValueError(f"{name}: expected finite FP32 {shape}, got {value.shape}/{value.dtype}")
|
||||
return value
|
||||
|
||||
|
||||
def compare(reference: np.ndarray, actual: np.ndarray) -> dict:
|
||||
original = reference.astype(np.float64).ravel()
|
||||
observed = actual.astype(np.float64).ravel()
|
||||
error = original - observed
|
||||
norm = float(np.linalg.norm(original))
|
||||
delta = float(np.linalg.norm(error))
|
||||
return {
|
||||
"relative_l2": delta / norm if norm else (0.0 if delta == 0 else None),
|
||||
"max_abs_error": float(np.max(np.abs(error))),
|
||||
"reference_rms": float(np.sqrt(np.mean(original * original))),
|
||||
"output_rms": float(np.sqrt(np.mean(observed * observed))),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source-onnx", type=Path, required=True)
|
||||
parser.add_argument("--fixture", type=Path, required=True)
|
||||
parser.add_argument("--cpu-reference", type=Path, required=True)
|
||||
parser.add_argument("--compile-report", type=Path, required=True)
|
||||
parser.add_argument("--inference-report", type=Path, required=True)
|
||||
parser.add_argument("--device-output", type=Path, required=True)
|
||||
parser.add_argument("--output-report", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
compiled = json.loads(args.compile_report.read_text(encoding="utf-8"))
|
||||
inferred = json.loads(args.inference_report.read_text(encoding="utf-8"))
|
||||
if compiled.get("status") != "SUCCESS" or inferred.get("status") != "SUCCESS":
|
||||
raise ValueError("compile and inference must both be terminal successes")
|
||||
if compiled.get("source_model_id") != "mq3xre4rq" or compiled.get("options") != COMPILE_OPTIONS:
|
||||
raise ValueError("compile source or options differ from the pinned talker artifact")
|
||||
if inferred.get("model_id") != compiled.get("target_model_id"):
|
||||
raise ValueError("inference did not use the audited target artifact")
|
||||
if inferred.get("input_dataset_id") != "d7dw6pgy2" or inferred.get("options") != "--compute_unit npu":
|
||||
raise ValueError("inference fixture or requested compute unit differs")
|
||||
if inferred.get("device") != compiled.get("device"):
|
||||
raise ValueError("compile/inference device metadata differs")
|
||||
|
||||
files = {
|
||||
name: file_record(path)
|
||||
for name, path in {
|
||||
"source_onnx": args.source_onnx,
|
||||
"fixture": args.fixture,
|
||||
"cpu_reference": args.cpu_reference,
|
||||
"device_output": args.device_output,
|
||||
}.items()
|
||||
}
|
||||
for name, expected in {
|
||||
"source_onnx": SOURCE_SHA256,
|
||||
"fixture": FIXTURE_SHA256,
|
||||
"cpu_reference": REFERENCE_SHA256,
|
||||
}.items():
|
||||
if files[name]["sha256"] != expected:
|
||||
raise ValueError(f"{name} hash differs from pinned source")
|
||||
|
||||
with np.load(args.fixture, allow_pickle=False) as fixture:
|
||||
if fixture.files != INPUT_NAMES:
|
||||
raise ValueError("fixture input names/order differ from the source contract")
|
||||
require_array(fixture, "x", (1, 1, 1024))
|
||||
for name in ("cos", "sin"):
|
||||
require_array(fixture, name, (1, 1, 1, 128))
|
||||
require_array(fixture, "mask", (1, 1, 1, 257))
|
||||
for name in INPUT_NAMES[4:]:
|
||||
require_array(fixture, name, (1, 8, 256, 128))
|
||||
|
||||
details = {}
|
||||
with np.load(args.cpu_reference, allow_pickle=False) as reference, np.load(
|
||||
args.device_output, allow_pickle=False
|
||||
) as output:
|
||||
if reference.files != OUTPUT_NAMES:
|
||||
raise ValueError("CPU reference output names/order differ from the source contract")
|
||||
expected_device_keys = [f"output_{index}__0" for index in range(len(OUTPUT_NAMES))]
|
||||
if output.files != expected_device_keys:
|
||||
raise ValueError("device output names/order differ from the compiled contract")
|
||||
for index, name in enumerate(OUTPUT_NAMES):
|
||||
shape = (1, 3072) if index == 0 else (1, 8, 1, 128)
|
||||
source_value = require_array(reference, name, shape)
|
||||
device_value = require_array(output, expected_device_keys[index], shape)
|
||||
details[name] = compare(source_value, device_value)
|
||||
if name == "logits":
|
||||
details[name]["cpu_top1"] = int(np.argmax(source_value[0]))
|
||||
details[name]["device_top1"] = int(np.argmax(device_value[0]))
|
||||
details[name]["top1_matches"] = bool(
|
||||
details[name]["cpu_top1"] == details[name]["device_top1"]
|
||||
)
|
||||
|
||||
report = {
|
||||
"scope": "one synthetic fixed-cache 28-layer Qwen3-TTS talker decode step; no complete TTS stream",
|
||||
"status": "component_executed_numeric_measured",
|
||||
"device": inferred["device"],
|
||||
"compile_job_id": compiled["job_id"],
|
||||
"inference_job_id": inferred["job_id"],
|
||||
"source_model_id": compiled["source_model_id"],
|
||||
"target_model_id": compiled["target_model_id"],
|
||||
"input_dataset_id": inferred["input_dataset_id"],
|
||||
"requested_run_options": inferred["options"],
|
||||
"files": files,
|
||||
"tensor_comparison": details,
|
||||
"cache_max_relative_l2": max(
|
||||
details[name]["relative_l2"] for name in OUTPUT_NAMES[1:]
|
||||
),
|
||||
"limits": [
|
||||
"The historical source export does not attest an exact checkpoint revision.",
|
||||
"The fixture is synthetic and has no token-to-audio quality tolerance.",
|
||||
"Requested NPU is not actual placement without a device profile.",
|
||||
"No predictor, vocoder, persistent cache loop or complete TTS stream was tested.",
|
||||
],
|
||||
}
|
||||
args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({
|
||||
"status": report["status"],
|
||||
"device": report["device"]["name"],
|
||||
"logits": details["logits"],
|
||||
"cache_max_relative_l2": report["cache_max_relative_l2"],
|
||||
}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a pinned Qwen3-TTS Code2Wav TFLite binary on local LiteRT CPU."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import platform
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from ai_edge_litert.interpreter import Interpreter
|
||||
|
||||
from audit_qwen_tts_vocoder_qualcomm import compare, read_output
|
||||
|
||||
|
||||
TFLITE_SHA256 = "29826da6ecc1ec2efc0e3589c3cebb42b20d7e20c0975a037d9f51625d6638cd"
|
||||
FIXTURE_SHA256 = "dfca9e8d2a724a568a31b6cd18ff6a52fdc205fd34e13cbf1cd75f0f7937e82d"
|
||||
REFERENCE_SHA256 = "8de78b94b59f01e49852f7706adbd2508a9976b8e2705cfd3b94d07bb3606cb1"
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
with path.open("rb") as source:
|
||||
return hashlib.file_digest(source, "sha256").hexdigest()
|
||||
|
||||
|
||||
def cpu_model() -> str:
|
||||
cpuinfo = Path("/proc/cpuinfo")
|
||||
if cpuinfo.exists():
|
||||
for line in cpuinfo.read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith("model name"):
|
||||
return line.partition(":")[2].strip()
|
||||
return platform.processor()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for name in ("model", "fixture", "reference", "output", "report"):
|
||||
parser.add_argument(f"--{name}", type=Path, required=True)
|
||||
parser.add_argument("--threads", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
if args.threads < 1:
|
||||
raise ValueError("threads must be positive")
|
||||
for name, expected in {
|
||||
"model": TFLITE_SHA256,
|
||||
"fixture": FIXTURE_SHA256,
|
||||
"reference": REFERENCE_SHA256,
|
||||
}.items():
|
||||
if sha256(getattr(args, name)) != expected:
|
||||
raise ValueError(f"{name} differs from the pinned artifact")
|
||||
|
||||
with np.load(args.fixture, allow_pickle=False) as data:
|
||||
if data.files != ["quantized"]:
|
||||
raise ValueError("unexpected input dataset layout")
|
||||
input_value = np.asarray(data["quantized"])
|
||||
if input_value.shape != (1, 512, 97) or input_value.dtype != np.float32:
|
||||
raise ValueError("expected one FP32 [1,512,97] input")
|
||||
|
||||
interpreter = Interpreter(model_path=str(args.model), num_threads=args.threads)
|
||||
interpreter.allocate_tensors()
|
||||
inputs = interpreter.get_input_details()
|
||||
outputs = interpreter.get_output_details()
|
||||
if len(inputs) != 1 or len(outputs) != 1 or inputs[0]["name"] != "quantized":
|
||||
raise ValueError("unexpected TFLite I/O contract")
|
||||
if tuple(inputs[0]["shape"]) != input_value.shape or inputs[0]["dtype"] != np.float32:
|
||||
raise ValueError("TFLite input shape/type differs")
|
||||
if tuple(outputs[0]["shape"]) != (1, 48000) or outputs[0]["dtype"] != np.float32:
|
||||
raise ValueError("TFLite output shape/type differs")
|
||||
|
||||
interpreter.set_tensor(inputs[0]["index"], input_value)
|
||||
started = time.monotonic()
|
||||
interpreter.invoke()
|
||||
elapsed = time.monotonic() - started
|
||||
actual = np.asarray(interpreter.get_tensor(outputs[0]["index"]))
|
||||
if actual.shape != (1, 48000) or actual.dtype != np.float32 or not np.isfinite(actual).all():
|
||||
raise ValueError("TFLite output violates finite waveform contract")
|
||||
np.savez_compressed(args.output, wav=actual)
|
||||
|
||||
source = read_output(args.reference, "wav")
|
||||
report = {
|
||||
"scope": "local x86-64 LiteRT CPU execution of one retained Code2Wav TFLite binary; no hosted-device timing or complete TTS",
|
||||
"status": "component_cpu_numeric_parity_on_one_fixture",
|
||||
"runtime": {"ai_edge_litert": importlib.metadata.version("ai-edge-litert"), "numpy": np.__version__},
|
||||
"host": {"system": platform.system(), "release": platform.release(), "machine": platform.machine(), "processor": cpu_model()},
|
||||
"threads": args.threads,
|
||||
"model_sha256": sha256(args.model),
|
||||
"fixture_sha256": sha256(args.fixture),
|
||||
"reference_sha256": sha256(args.reference),
|
||||
"output_sha256": sha256(args.output),
|
||||
"source_onnx_cpu_vs_litert_cpu": compare(source, actual),
|
||||
"single_invoke_seconds_not_a_profile": elapsed,
|
||||
"limits": [
|
||||
"Local x86-64 CPU execution cannot attest SA8775P or RB3 device CPU behavior.",
|
||||
"LiteRT CPU output is not evidence that the Android GPU delegate is correct.",
|
||||
"One fixed vocoder fixture has no listening-quality tolerance or complete-stream gate.",
|
||||
],
|
||||
}
|
||||
args.report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"status": report["status"], "comparison": report["source_onnx_cpu_vs_litert_cpu"]}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,5 +1,7 @@
|
||||
# 2026-09-24 Qualcomm component expansion
|
||||
|
||||
The [Qwen3-TTS vocoder study](evidence/qwen_tts_vocoder_qualcomm/README.md) reuses a retained fixed-shape Code2Wav export and historical Workbench input to test numerical behavior on SA8775P and other Qualcomm settings. The fresh SA8775P GPU inference produces a saturated waveform; RB3 produces an unsaturated but divergent waveform. Neither is numerically qualified as a usable TTS path. X Elite ONNX CPU output matches the local ONNX Runtime reference on the retained fixture; its profile and DirectML retry remain pending. Other Workbench jobs for RB3 profiling and SA8775P controls are also pending; their submission IDs are retained for completion. No complete on-device TTS stream or model-quality qualification is claimed.
|
||||
The [Qwen3-TTS vocoder study](evidence/qwen_tts_vocoder_qualcomm/README.md) reuses a retained fixed-shape Code2Wav export and historical Workbench input to test numerical behavior on SA8775P and other Qualcomm settings. The fresh SA8775P GPU inference produces a saturated waveform; RB3 produces an unsaturated but divergent waveform. Neither is numerically qualified as a usable TTS path. X Elite ONNX CPU output matches the local ONNX Runtime reference on the retained fixture; its 94-sample component p50/p95 was 6.384/6.644 s with all execution-detail rows on CPU. Its DirectML retry remains pending. Other Workbench jobs for RB3 profiling and SA8775P controls are also pending; their submission IDs are retained for completion. No complete on-device TTS stream or model-quality qualification is claimed.
|
||||
|
||||
A local HX370 x86-64 LiteRT CPU control of the identical SA/RB3 TFLite binary matched the original ONNX CPU waveform within 3.45e-6 relative L2. The SA saturation is therefore not universal to the converted artifact; hosted CPU and GPU precision controls are still needed to isolate its device/runtime cause.
|
||||
|
||||
The [talker decode-step attempt](evidence/qwen_tts_talker_qualcomm/README.md) pins a separate 28-layer source and synthetic cache fixture. Historical FP16 QNN context-binary compiles passed for S24, S25, SA8775P and X Elite, with same-fixture inference pending; RB3 rejected that floating-point-input artifact. Compile success is not a numerical or end-to-end pass.
|
||||
+2
@@ -6,4 +6,6 @@ The deterministic synthetic input dataset `d7dw6pgy2` has SHA256 `0bf538e1b64dfc
|
||||
|
||||
Historical FP16 QNN context-binary [S24](s24/compile_report.json), [S25](s25/compile_report.json), [SA8775P](sa8775p/compile_report.json), and [Snapdragon X Elite](xelite/compile_report.json) compile jobs succeeded. Their same-fixture NPU inference jobs were [submitted](s24/inference_submission.json), [submitted](s25/inference_submission.json), [submitted](sa8775p/inference_submission.json), and [submitted](xelite/inference_submission.json), respectively; none had a terminal result at this snapshot. The [RB3 compile](rb3/compile_report.json) rejected the floating-point input on this device. These are artifact-specific compile dispositions, not full-model support claims.
|
||||
|
||||
The [collector](../../../../experiments/collect_aihub_component.py) retrieves terminal inference outputs without resubmission. The [tensor auditor](../../../../experiments/audit_qwen_tts_talker_qualcomm.py) enforces the source, fixture, compile target, exact device, output shape and finiteness contracts, and reports logits top-1 plus relative error for all 56 new K/V tensors. It does not convert those metrics to a quality pass without an explicit token/audio tolerance. Its 57-output contract was exercised against the pinned CPU reference as a local self-check; that self-check is not device evidence.
|
||||
|
||||
Next: retrieve terminal inference tensors, compare logits and every K/V output against the pinned CPU reference with an explicit tolerance, inspect actual placement, then profile a quality-passing decode step. Full device-local talker/predictor/vocoder streaming and power/memory qualification remain separate gates.
|
||||
+4
-2
@@ -8,10 +8,12 @@ The [S25 historical inference](s25_reference_inference.json) on the same fixture
|
||||
|
||||
The fresh [RB3 inference](rb3/inference_report.json) on the same fixture returned a finite, **unsaturated** waveform ([output](rb3/device_output.npz)). Its [comparison](rb3/inference_comparison.json) against original ONNX CPU was **15.73% relative L2 / 16.06 dB SNR**, still without a listening-quality threshold. Thus the bitwise identical SA and RB3 TFLite target downloads produce materially different outputs on their respective devices; target file identity does not establish runtime numerical parity. The fresh RB3 profile was still pending at this snapshot.
|
||||
|
||||
A separate [local LiteRT 2.2.0 CPU control](local_litert_cpu/report.json) ran that **same pinned TFLite binary** on the WSL x86-64 HX370 host with four interpreter threads. Its [waveform](local_litert_cpu/device_output.npz) was finite and unsaturated, within **3.45e-6 relative L2 / 109.23 dB SNR** of the original ONNX Runtime CPU output. The [reproducer](../../../../experiments/probe_qwen_tts_vocoder_litert_cpu.py) checks binary/fixture/reference hashes and I/O contracts. This narrows the SA saturation to hosted runtime/device behavior rather than a universally bad TFLite conversion. The one local invoke is not a comparable performance profile; it does not prove either embedded device's CPU route or identify the specific delegate operation at fault.
|
||||
|
||||
The re-fetched *historical* [SA8775P profile](sa8775p/historical_profile_report.json) has 100 component samples, nearest-rank p50/p95 **1.751/1.874 s**, and 403 CPU plus 335 GPU execution-detail rows. The *historical* [RB3 profile](rb3/historical_profile_report.json) has 100 samples, p50/p95 **3.143/3.189 s**, also with 403 CPU plus 335 GPU rows. Their previously quoted 1.701/3.107 s estimated-time fields are sample minima, not medians. Both profiles exclude talker/predictor, stage handoff, streaming and full-request time.
|
||||
|
||||
The X Elite [ONNX CPU inference](xelite/cpu/inference_report.json) succeeded on the pinned fixture. Its [waveform comparison](xelite/cpu/inference_comparison.json) to the original local ONNX Runtime CPU output was **8.09e-7 relative L2 / 121.84 dB SNR**, with no saturation. This is near-source component parity on one fixture; the CPU profile and listening-quality gate remain open. The first X Elite [GPU inference](xelite/gpu/inference_report.json) and [profile](xelite/gpu/profile_report.json) jobs failed because Workbench rejects specifying both compute unit and ONNX execution provider; the [DirectML retry](xelite/directml/inference_submission.json) specifies DirectML alone. The [X Elite ONNX compile](xelite/compile_report.json) passed after the older TFLite target was rejected on Windows ARM.
|
||||
The X Elite [ONNX CPU inference](xelite/cpu/inference_report.json) succeeded on the pinned fixture. Its [waveform comparison](xelite/cpu/inference_comparison.json) to the original local ONNX Runtime CPU output was **8.09e-7 relative L2 / 121.84 dB SNR**, with no saturation. The fresh [CPU profile](xelite/cpu/profile_report.json) returned **94** component samples, nearest-rank p50/p95 **6.384/6.644 s**, with all **634** execution-detail rows on CPU; the service's 5.917 s estimated-time field is the sample minimum. The [full audit](xelite/cpu/audit_report.json) pins source, fixture, target, waveform, placement and percentiles. This is near-source component parity on one fixture, without a listening-quality or complete-stream gate. The first X Elite [GPU inference](xelite/gpu/inference_report.json) and [profile](xelite/gpu/profile_report.json) jobs failed because Workbench rejects specifying both compute unit and ONNX execution provider; the [DirectML retry](xelite/directml/inference_submission.json) specifies DirectML alone. The [X Elite ONNX compile](xelite/compile_report.json) passed after the older TFLite target was rejected on Windows ARM.
|
||||
|
||||
The fresh [RB3 profile](rb3/profile_submission.json), [SA CPU](sa8775p/cpu_control/inference_submission.json) and [FP32-preserving GPU](sa8775p/gpu_fp32/inference_submission.json) controls, X Elite [DirectML retry](xelite/directml/inference_submission.json) and [CPU profile](xelite/cpu/profile_submission.json) remained pending in this snapshot. Pending jobs have no numerical or latency conclusion. The [collector](../../../../experiments/collect_aihub_component.py) retrieves their terminal reports without resubmitting jobs, and the [audit script](../../../../experiments/audit_qwen_tts_vocoder_qualcomm.py) checks source/fixture hashes, device and model IDs, actual placement, waveform error, saturation and sample percentiles.
|
||||
The fresh [RB3 profile](rb3/profile_submission.json), [SA CPU](sa8775p/cpu_control/inference_submission.json) and [FP32-preserving GPU](sa8775p/gpu_fp32/inference_submission.json) controls, and X Elite [DirectML retry](xelite/directml/inference_submission.json) remained pending in this snapshot. Pending jobs have no numerical or latency conclusion. The [collector](../../../../experiments/collect_aihub_component.py) retrieves their terminal reports without resubmitting jobs, and the [audit script](../../../../experiments/audit_qwen_tts_vocoder_qualcomm.py) checks source/fixture hashes, device and model IDs, actual placement, waveform error, saturation and sample percentiles.
|
||||
|
||||
This is one fixed vocoder component with a retained input, not complete Qwen3-TTS. The next gates are a quality-passing SA/RB3/X Elite target, talker and code predictor numerical checks, device-resident handoff/co-residency, playable streaming, cancellation, memory admission and sustained power/thermal profiling.
|
||||
BIN
Binary file not shown.
+34
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"scope": "local x86-64 LiteRT CPU execution of one retained Code2Wav TFLite binary; no hosted-device timing or complete TTS",
|
||||
"status": "component_cpu_numeric_parity_on_one_fixture",
|
||||
"runtime": {
|
||||
"ai_edge_litert": "2.2.0",
|
||||
"numpy": "2.5.3"
|
||||
},
|
||||
"host": {
|
||||
"system": "Linux",
|
||||
"release": "6.18.33.2-microsoft-standard-WSL2",
|
||||
"machine": "x86_64",
|
||||
"processor": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M"
|
||||
},
|
||||
"threads": 4,
|
||||
"model_sha256": "29826da6ecc1ec2efc0e3589c3cebb42b20d7e20c0975a037d9f51625d6638cd",
|
||||
"fixture_sha256": "dfca9e8d2a724a568a31b6cd18ff6a52fdc205fd34e13cbf1cd75f0f7937e82d",
|
||||
"reference_sha256": "8de78b94b59f01e49852f7706adbd2508a9976b8e2705cfd3b94d07bb3606cb1",
|
||||
"output_sha256": "c19524684abaf5da5c5f2c539a536ddba3ecc5e094c252e1dadc2804f0b482c7",
|
||||
"source_onnx_cpu_vs_litert_cpu": {
|
||||
"relative_l2": 3.454229291440569e-06,
|
||||
"snr_db": 109.23297674763376,
|
||||
"max_abs": 5.692243576049805e-06,
|
||||
"cosine": 0.9999999999940402,
|
||||
"reference_rms": 0.08373439205485074,
|
||||
"output_rms": 0.08373440156017416,
|
||||
"output_saturated_fraction_abs_ge_0_999": 0.0
|
||||
},
|
||||
"single_invoke_seconds_not_a_profile": 1.6104475790052675,
|
||||
"limits": [
|
||||
"Local x86-64 CPU execution cannot attest SA8775P or RB3 device CPU behavior.",
|
||||
"LiteRT CPU output is not evidence that the Android GPU delegate is correct.",
|
||||
"One fixed vocoder fixture has no listening-quality tolerance or complete-stream gate."
|
||||
]
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"scope": "one Qwen3-TTS code2wav 48,000-sample component on one retained fixture; no complete TTS stream",
|
||||
"status": "component_executed_quality_unqualified",
|
||||
"device": {
|
||||
"name": "Snapdragon X Elite CRD",
|
||||
"os": "11",
|
||||
"attributes": [
|
||||
"os:windows",
|
||||
"format:compute",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-windows",
|
||||
"chipset:qualcomm-snapdragon-x-elite",
|
||||
"chipset:sc8380xp",
|
||||
"hexagon:v73",
|
||||
"soc-model:60",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"compile_job_id": "j5qlyz8op",
|
||||
"inference_job_id": "j5wl78nmp",
|
||||
"profile_job_id": "jg9zmke8p",
|
||||
"target_model_id": "mnz079ezm",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"s25_prior_inference_job_id": "jpx428ojg",
|
||||
"requested_run_options": "--compute_unit cpu",
|
||||
"files": {
|
||||
"source_onnx": {
|
||||
"bytes": 423655047,
|
||||
"sha256": "7c07666229c6d404894132e22f784525e3212a49efad55ddd532eda5286c2b2b"
|
||||
},
|
||||
"fixture": {
|
||||
"bytes": 129721,
|
||||
"sha256": "dfca9e8d2a724a568a31b6cd18ff6a52fdc205fd34e13cbf1cd75f0f7937e82d"
|
||||
},
|
||||
"cpu_output": {
|
||||
"bytes": 179942,
|
||||
"sha256": "8de78b94b59f01e49852f7706adbd2508a9976b8e2705cfd3b94d07bb3606cb1"
|
||||
},
|
||||
"s25_output": {
|
||||
"bytes": 115235,
|
||||
"sha256": "4a26486d5e84a6e22d16f9b26f995b8fe645c29bbd70b103615ec35a0fcb10a0"
|
||||
},
|
||||
"device_output": {
|
||||
"bytes": 179943,
|
||||
"sha256": "de7e4605bc6508c19c22ca3e3f5d2d67c1002908aa1e3cebfe7f40bd040f304a"
|
||||
}
|
||||
},
|
||||
"cpu_source_vs_device": {
|
||||
"relative_l2": 8.086912399814565e-07,
|
||||
"snr_db": 121.84434522578867,
|
||||
"max_abs": 9.5367431640625e-07,
|
||||
"cosine": 0.9999999999996769,
|
||||
"reference_rms": 0.08373439205485074,
|
||||
"output_rms": 0.0837343848397309,
|
||||
"output_saturated_fraction_abs_ge_0_999": 0.0
|
||||
},
|
||||
"s25_gpu_vs_device": {
|
||||
"relative_l2": 0.04236831072350974,
|
||||
"snr_db": 27.45917703014474,
|
||||
"max_abs": 0.10591856762766838,
|
||||
"cosine": 0.9991021841139998,
|
||||
"reference_rms": 0.08376787228276501,
|
||||
"output_rms": 0.0837343848397309,
|
||||
"output_saturated_fraction_abs_ge_0_999": 0.0
|
||||
},
|
||||
"profile": {
|
||||
"sample_count": 94,
|
||||
"sample_min_us": 5916879,
|
||||
"nearest_rank_p50_us": 6384115,
|
||||
"nearest_rank_p95_us": 6644231,
|
||||
"reported_estimated_inference_time_us": 5916879,
|
||||
"reported_peak_memory_bytes": 1340260352,
|
||||
"compute_unit_row_counts": {
|
||||
"CPU": 634
|
||||
}
|
||||
},
|
||||
"limits": [
|
||||
"Historical source export does not attest its exact checkpoint revision.",
|
||||
"This fixed component waveform has no listening or task-quality tolerance.",
|
||||
"Hosted component profiling excludes talker, predictor, stage handoff, streaming, admission and sustained power/thermal behavior."
|
||||
]
|
||||
}
|
||||
+4627
File diff suppressed because it is too large.
Load diff
@@ -12,7 +12,9 @@ On a hosted [Galaxy S24 Spark attention-layer experiment](../../../e2e_expansion
|
||||
|
||||
The same W8A16 layer also ran on hosted [Snapdragon X Elite CRD](../../../e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md) and [SA8775P ADP](../../../e2e_expansion_20260923/evidence/spark_sa8775_full_attention/README.md) NPUs, with pinned-fixture outputs identical to S24/S25; the 100-sample component p50/p95 was 1.433/1.606 ms and 1.837/2.079 ms, respectively. On [RB3 Gen 2 / QCS6490](../../../e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md), that W8A16 graph failed QNN DLC loading even after target-specific recompilation. An FP32 ONNX CPU layer then passed with source-relative L2 below 9e-7, while a calibrated W8A8 NPU layer executed but had 20.9% hidden-state relative L2 and remains numerically unqualified. These hosted component outcomes do not establish full generation or comparable device speed.
|
||||
|
||||
A [Qwen3-TTS Code2Wav follow-up](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) recovered source and target artifacts, a retained fixture, and raw historical profiles. On SA8775P, fresh GPU-requested inference saturated 96.63% of waveform samples and differed from original ONNX CPU by 11.832 relative L2; this tested vocoder path is numerically unqualified. Fresh RB3 inference was unsaturated but differed from CPU by 15.73% relative L2 (16.06 dB SNR), also without a listening-quality gate. Historical SA8775P/RB3 profile p50/p95 values were 1.751/1.874 s and 3.143/3.189 s, respectively, with both CPU and GPU rows; earlier estimated-time fields were minima. X Elite ONNX CPU vocoder inference passed on the same fixture with 8.09e-7 relative L2 versus local ONNX CPU; its profile and DirectML retry were pending. None is a complete TTS stream.
|
||||
A [Qwen3-TTS Code2Wav follow-up](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) recovered source and target artifacts, a retained fixture, and raw historical profiles. On SA8775P, fresh GPU-requested inference saturated 96.63% of waveform samples and differed from original ONNX CPU by 11.832 relative L2; this tested vocoder path is numerically unqualified. Fresh RB3 inference was unsaturated but differed from CPU by 15.73% relative L2 (16.06 dB SNR), also without a listening-quality gate. Historical SA8775P/RB3 profile p50/p95 values were 1.751/1.874 s and 3.143/3.189 s, respectively, with both CPU and GPU rows; earlier estimated-time fields were minima. X Elite ONNX CPU vocoder inference passed on the same fixture with 8.09e-7 relative L2 versus local ONNX CPU; its 94-sample component p50/p95 was 6.384/6.644 s with all execution-detail rows on CPU. DirectML retry was pending. None is a complete TTS stream.
|
||||
|
||||
The identical SA/RB3 TFLite binary matched original ONNX CPU within 3.45e-6 relative L2 in a separate local HX370 x86-64 LiteRT CPU control. This narrows the SA waveform saturation to hosted device/runtime behavior, without proving either embedded CPU route or a specific GPU-delegate fault.
|
||||
|
||||
A separate [Qwen3-TTS talker decode-step attempt](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) recovered a 28-layer ONNX export and synthetic fixed-cache fixture. Historical FP16 QNN context-binary compiles passed for S24, S25, SA8775P and X Elite, but the same-fixture inference jobs were pending. An RB3 compile rejected floating-point input. No talker runtime or numerical pass is implied by those compile outcomes.
|
||||
|
||||
@@ -81,7 +83,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | InternVLA-A1 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Spark-X2.5 | COMPONENT_NPU_INFERENCE_PROFILED; NOT_E2E_PROFILEABLE | A real Spark-X2.5-1.7B one-layer W8A16 QDQ attention artifact, compiled as QNN DLC, ran on the exact hosted Snapdragon X Elite CRD (Windows 11 ARM64). Inference returned finite hidden/new-K/new-V tensors bitwise identical to the prior S24/S25 NPU outputs on the same fixture; the profile attributed all 73 execution-detail rows to NPU. Across 100 component samples, nearest-rank p50/p95 was 1.433/1.606 ms; the 1.346 ms estimated-time field is the minimum. Hidden/K/V relative L2 versus original FP32 ONNX CPU was 1.351%/0.377%/0.650%, without a next-token quality tolerance. There is no complete 28-layer prefill/decode/KV-ring/sampling loop, resident Omni controller or complete-request timing. | Qualify a complete X Elite resident artifact/controller with state, token quality, memory admission/cancel and sustained power; [X Elite NPU raw evidence](../../../e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md). |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_ONNX_CPU_VOCODER_PARITY_PASS; NOT_E2E_PROFILEABLE | The fresh ONNX vocoder inference on the exact Windows 11 ARM64 X Elite CRD requested CPU and returned an unsaturated waveform on the pinned fixture, matching local ONNX Runtime CPU with 8.09e-7 relative L2 and 121.84 dB SNR. This is one component output, without a listening-quality gate. CPU profile and corrected DirectML inference/profile jobs were pending; the initial DirectML submissions failed because mutually exclusive run options were specified. Historical QNN talker profiling lacks a same-fixture numerical or full-pipeline gate. The older TFLite vocoder target was rejected on Windows ARM; no complete stream exists. | Audit CPU profile and DirectML retry, qualify talker/predictor and complete local streaming, memory and sustained behavior; [vocoder evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md). |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_ONNX_CPU_VOCODER_PARITY_PROFILED; NOT_E2E_PROFILEABLE | The fresh ONNX vocoder inference on the exact Windows 11 ARM64 X Elite CRD requested CPU and returned an unsaturated waveform on the pinned fixture, matching local ONNX Runtime CPU with 8.09e-7 relative L2 and 121.84 dB SNR. Its 94-sample component profile had nearest-rank p50/p95 6.384/6.644 s and all 634 execution-detail rows on CPU; 5.917 s estimated time is the sample minimum. This is one component, without a listening-quality gate. Corrected DirectML inference/profile jobs were pending; the initial DirectML submissions failed because mutually exclusive run options were specified. Historical QNN talker profiling lacks a same-fixture numerical or full-pipeline gate. The older TFLite vocoder target was rejected on Windows ARM; no complete stream exists. | Audit DirectML retry, qualify talker/predictor and complete local streaming, memory and sustained behavior; [vocoder evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md). |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
|
||||
@@ -15,7 +15,9 @@ remain open; the 2026-09-22 audit remains the historical baseline.
|
||||
|
||||
The [Spark hosted Qualcomm component expansion](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/README.md) advances M1 only to one real fixed-shape decoder attention layer. The same W8A16 QNN DLC ran on S24, Snapdragon X Elite CRD and SA8775P ADP NPUs with identical outputs on one pinned fixture; S25 had prior separate component evidence. On [RB3 Gen 2](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md), the tested W8A16 graph failed QNN loading even after an exact-device compile. FP32 ONNX ran on CPU with near-source parity, while calibrated W8A8 ran on NPU but had 20.9% hidden-state relative L2 and remains numerically unqualified. These are component C results, not full M1: device-local 28-layer prefill/continuous decode, KV/ring, sampling, token quality, admission/cancellation, complete-request timing and sustained power/thermal gates remain open. AI Hub is the test facility, not a deployment dependency.
|
||||
|
||||
The [Qwen3-TTS Code2Wav Qualcomm follow-up](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) adds a retained same-fixture numerical check to M2. A fresh SA8775P GPU-requested output saturated 96.63% of samples and differs from original ONNX CPU by 11.832 relative L2, so the tested vocoder target fails the numerical gate despite historical component profiling. Historical SA8775P and RB3 vocoder profiles have mixed CPU/GPU placement; their estimated-time fields were minima. RB3 TFLite GPU-requested inference returned an unsaturated but 15.73%-relative-L2 divergent waveform; its fresh profile was pending. X Elite ONNX CPU inference matched the local ONNX Runtime output with 8.09e-7 relative L2, while its profile, DirectML retry and SA controls remained pending. M2 still requires a quality-passing talker/predictor/vocoder set, device-local co-residency and handoff, playable streaming and sustained memory/power validation.
|
||||
The [Qwen3-TTS Code2Wav Qualcomm follow-up](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) adds a retained same-fixture numerical check to M2. A fresh SA8775P GPU-requested output saturated 96.63% of samples and differs from original ONNX CPU by 11.832 relative L2, so the tested vocoder target fails the numerical gate despite historical component profiling. Historical SA8775P and RB3 vocoder profiles have mixed CPU/GPU placement; their estimated-time fields were minima. RB3 TFLite GPU-requested inference returned an unsaturated but 15.73%-relative-L2 divergent waveform; its fresh profile was pending. X Elite ONNX CPU inference matched the local ONNX Runtime output with 8.09e-7 relative L2; its 94-sample component profile was p50/p95 6.384/6.644 s with all execution-detail rows on CPU. DirectML retry and SA controls remained pending. M2 still requires a quality-passing talker/predictor/vocoder set, device-local co-residency and handoff, playable streaming and sustained memory/power validation.
|
||||
|
||||
The identical SA/RB3 TFLite vocoder binary also matched original ONNX CPU within 3.45e-6 relative L2 on a separate local HX370 x86-64 LiteRT 2.2.0 CPU control. That narrows the SA failure to hosted device/runtime behavior, but cannot establish either embedded CPU route or the cause within the GPU delegate. The pending SA CPU and FP32-preserving GPU controls are the next isolation gate.
|
||||
|
||||
A separate [28-layer talker decode-step attempt](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) retains a synthetic fixed-cache fixture and CPU reference. Historical FP16 QNN context-binary compiles passed for S24, S25, SA8775P and X Elite; their same-fixture NPU inference was pending at this snapshot. RB3 rejected that floating-point-input artifact. None advances M2 to numerical talker or full-stream qualification yet.
|
||||
|
||||
|
||||
Reference in new issue
Block a user