Audit X Elite CPU vocoder and isolate TFLite conversion parity
This commit is contained in:
1 parent
e4a4477929
commit
ea373e3cbf
11 files changed
+5025
-6
No files matched your search
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit one hosted Qwen3-TTS talker decode step against pinned ONNX CPU tensors."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
SOURCE_SHA256 = "41feb08517678e94d9301e992d0df610fc60e7c9faba5600703298c86554fd22"
|
||||
FIXTURE_SHA256 = "0bf538e1b64dfc9737dc4f925d61c6e4ffa5075a5dcd7afb1e85ac3bd2d9afea"
|
||||
REFERENCE_SHA256 = "ee0ead70e33c02c7d3d6bcb757c5cf0fcbb9eb6fc75c34abfbe8a4479657d5be"
|
||||
COMPILE_OPTIONS = "--target_runtime qnn_context_binary --quantize_full_type float16"
|
||||
INPUT_NAMES = ["x", "cos", "sin", "mask"] + [
|
||||
name for layer in range(28) for name in (f"k_cache_{layer}", f"v_cache_{layer}")
|
||||
]
|
||||
OUTPUT_NAMES = ["logits"] + [
|
||||
name for layer in range(28) for name in (f"k_new_{layer}", f"v_new_{layer}")
|
||||
]
|
||||
|
||||
|
||||
def file_record(path: Path) -> dict:
|
||||
with path.open("rb") as source:
|
||||
digest = hashlib.file_digest(source, "sha256").hexdigest()
|
||||
return {"bytes": path.stat().st_size, "sha256": digest}
|
||||
|
||||
|
||||
def require_array(data: np.lib.npyio.NpzFile, name: str, shape: tuple[int, ...]) -> np.ndarray:
|
||||
value = np.asarray(data[name])
|
||||
if value.shape != shape or value.dtype != np.float32 or not np.isfinite(value).all():
|
||||
raise ValueError(f"{name}: expected finite FP32 {shape}, got {value.shape}/{value.dtype}")
|
||||
return value
|
||||
|
||||
|
||||
def compare(reference: np.ndarray, actual: np.ndarray) -> dict:
|
||||
original = reference.astype(np.float64).ravel()
|
||||
observed = actual.astype(np.float64).ravel()
|
||||
error = original - observed
|
||||
norm = float(np.linalg.norm(original))
|
||||
delta = float(np.linalg.norm(error))
|
||||
return {
|
||||
"relative_l2": delta / norm if norm else (0.0 if delta == 0 else None),
|
||||
"max_abs_error": float(np.max(np.abs(error))),
|
||||
"reference_rms": float(np.sqrt(np.mean(original * original))),
|
||||
"output_rms": float(np.sqrt(np.mean(observed * observed))),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source-onnx", type=Path, required=True)
|
||||
parser.add_argument("--fixture", type=Path, required=True)
|
||||
parser.add_argument("--cpu-reference", type=Path, required=True)
|
||||
parser.add_argument("--compile-report", type=Path, required=True)
|
||||
parser.add_argument("--inference-report", type=Path, required=True)
|
||||
parser.add_argument("--device-output", type=Path, required=True)
|
||||
parser.add_argument("--output-report", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
compiled = json.loads(args.compile_report.read_text(encoding="utf-8"))
|
||||
inferred = json.loads(args.inference_report.read_text(encoding="utf-8"))
|
||||
if compiled.get("status") != "SUCCESS" or inferred.get("status") != "SUCCESS":
|
||||
raise ValueError("compile and inference must both be terminal successes")
|
||||
if compiled.get("source_model_id") != "mq3xre4rq" or compiled.get("options") != COMPILE_OPTIONS:
|
||||
raise ValueError("compile source or options differ from the pinned talker artifact")
|
||||
if inferred.get("model_id") != compiled.get("target_model_id"):
|
||||
raise ValueError("inference did not use the audited target artifact")
|
||||
if inferred.get("input_dataset_id") != "d7dw6pgy2" or inferred.get("options") != "--compute_unit npu":
|
||||
raise ValueError("inference fixture or requested compute unit differs")
|
||||
if inferred.get("device") != compiled.get("device"):
|
||||
raise ValueError("compile/inference device metadata differs")
|
||||
|
||||
files = {
|
||||
name: file_record(path)
|
||||
for name, path in {
|
||||
"source_onnx": args.source_onnx,
|
||||
"fixture": args.fixture,
|
||||
"cpu_reference": args.cpu_reference,
|
||||
"device_output": args.device_output,
|
||||
}.items()
|
||||
}
|
||||
for name, expected in {
|
||||
"source_onnx": SOURCE_SHA256,
|
||||
"fixture": FIXTURE_SHA256,
|
||||
"cpu_reference": REFERENCE_SHA256,
|
||||
}.items():
|
||||
if files[name]["sha256"] != expected:
|
||||
raise ValueError(f"{name} hash differs from pinned source")
|
||||
|
||||
with np.load(args.fixture, allow_pickle=False) as fixture:
|
||||
if fixture.files != INPUT_NAMES:
|
||||
raise ValueError("fixture input names/order differ from the source contract")
|
||||
require_array(fixture, "x", (1, 1, 1024))
|
||||
for name in ("cos", "sin"):
|
||||
require_array(fixture, name, (1, 1, 1, 128))
|
||||
require_array(fixture, "mask", (1, 1, 1, 257))
|
||||
for name in INPUT_NAMES[4:]:
|
||||
require_array(fixture, name, (1, 8, 256, 128))
|
||||
|
||||
details = {}
|
||||
with np.load(args.cpu_reference, allow_pickle=False) as reference, np.load(
|
||||
args.device_output, allow_pickle=False
|
||||
) as output:
|
||||
if reference.files != OUTPUT_NAMES:
|
||||
raise ValueError("CPU reference output names/order differ from the source contract")
|
||||
expected_device_keys = [f"output_{index}__0" for index in range(len(OUTPUT_NAMES))]
|
||||
if output.files != expected_device_keys:
|
||||
raise ValueError("device output names/order differ from the compiled contract")
|
||||
for index, name in enumerate(OUTPUT_NAMES):
|
||||
shape = (1, 3072) if index == 0 else (1, 8, 1, 128)
|
||||
source_value = require_array(reference, name, shape)
|
||||
device_value = require_array(output, expected_device_keys[index], shape)
|
||||
details[name] = compare(source_value, device_value)
|
||||
if name == "logits":
|
||||
details[name]["cpu_top1"] = int(np.argmax(source_value[0]))
|
||||
details[name]["device_top1"] = int(np.argmax(device_value[0]))
|
||||
details[name]["top1_matches"] = bool(
|
||||
details[name]["cpu_top1"] == details[name]["device_top1"]
|
||||
)
|
||||
|
||||
report = {
|
||||
"scope": "one synthetic fixed-cache 28-layer Qwen3-TTS talker decode step; no complete TTS stream",
|
||||
"status": "component_executed_numeric_measured",
|
||||
"device": inferred["device"],
|
||||
"compile_job_id": compiled["job_id"],
|
||||
"inference_job_id": inferred["job_id"],
|
||||
"source_model_id": compiled["source_model_id"],
|
||||
"target_model_id": compiled["target_model_id"],
|
||||
"input_dataset_id": inferred["input_dataset_id"],
|
||||
"requested_run_options": inferred["options"],
|
||||
"files": files,
|
||||
"tensor_comparison": details,
|
||||
"cache_max_relative_l2": max(
|
||||
details[name]["relative_l2"] for name in OUTPUT_NAMES[1:]
|
||||
),
|
||||
"limits": [
|
||||
"The historical source export does not attest an exact checkpoint revision.",
|
||||
"The fixture is synthetic and has no token-to-audio quality tolerance.",
|
||||
"Requested NPU is not actual placement without a device profile.",
|
||||
"No predictor, vocoder, persistent cache loop or complete TTS stream was tested.",
|
||||
],
|
||||
}
|
||||
args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({
|
||||
"status": report["status"],
|
||||
"device": report["device"]["name"],
|
||||
"logits": details["logits"],
|
||||
"cache_max_relative_l2": report["cache_max_relative_l2"],
|
||||
}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a pinned Qwen3-TTS Code2Wav TFLite binary on local LiteRT CPU."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import platform
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from ai_edge_litert.interpreter import Interpreter
|
||||
|
||||
from audit_qwen_tts_vocoder_qualcomm import compare, read_output
|
||||
|
||||
|
||||
TFLITE_SHA256 = "29826da6ecc1ec2efc0e3589c3cebb42b20d7e20c0975a037d9f51625d6638cd"
|
||||
FIXTURE_SHA256 = "dfca9e8d2a724a568a31b6cd18ff6a52fdc205fd34e13cbf1cd75f0f7937e82d"
|
||||
REFERENCE_SHA256 = "8de78b94b59f01e49852f7706adbd2508a9976b8e2705cfd3b94d07bb3606cb1"
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
with path.open("rb") as source:
|
||||
return hashlib.file_digest(source, "sha256").hexdigest()
|
||||
|
||||
|
||||
def cpu_model() -> str:
|
||||
cpuinfo = Path("/proc/cpuinfo")
|
||||
if cpuinfo.exists():
|
||||
for line in cpuinfo.read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith("model name"):
|
||||
return line.partition(":")[2].strip()
|
||||
return platform.processor()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for name in ("model", "fixture", "reference", "output", "report"):
|
||||
parser.add_argument(f"--{name}", type=Path, required=True)
|
||||
parser.add_argument("--threads", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
if args.threads < 1:
|
||||
raise ValueError("threads must be positive")
|
||||
for name, expected in {
|
||||
"model": TFLITE_SHA256,
|
||||
"fixture": FIXTURE_SHA256,
|
||||
"reference": REFERENCE_SHA256,
|
||||
}.items():
|
||||
if sha256(getattr(args, name)) != expected:
|
||||
raise ValueError(f"{name} differs from the pinned artifact")
|
||||
|
||||
with np.load(args.fixture, allow_pickle=False) as data:
|
||||
if data.files != ["quantized"]:
|
||||
raise ValueError("unexpected input dataset layout")
|
||||
input_value = np.asarray(data["quantized"])
|
||||
if input_value.shape != (1, 512, 97) or input_value.dtype != np.float32:
|
||||
raise ValueError("expected one FP32 [1,512,97] input")
|
||||
|
||||
interpreter = Interpreter(model_path=str(args.model), num_threads=args.threads)
|
||||
interpreter.allocate_tensors()
|
||||
inputs = interpreter.get_input_details()
|
||||
outputs = interpreter.get_output_details()
|
||||
if len(inputs) != 1 or len(outputs) != 1 or inputs[0]["name"] != "quantized":
|
||||
raise ValueError("unexpected TFLite I/O contract")
|
||||
if tuple(inputs[0]["shape"]) != input_value.shape or inputs[0]["dtype"] != np.float32:
|
||||
raise ValueError("TFLite input shape/type differs")
|
||||
if tuple(outputs[0]["shape"]) != (1, 48000) or outputs[0]["dtype"] != np.float32:
|
||||
raise ValueError("TFLite output shape/type differs")
|
||||
|
||||
interpreter.set_tensor(inputs[0]["index"], input_value)
|
||||
started = time.monotonic()
|
||||
interpreter.invoke()
|
||||
elapsed = time.monotonic() - started
|
||||
actual = np.asarray(interpreter.get_tensor(outputs[0]["index"]))
|
||||
if actual.shape != (1, 48000) or actual.dtype != np.float32 or not np.isfinite(actual).all():
|
||||
raise ValueError("TFLite output violates finite waveform contract")
|
||||
np.savez_compressed(args.output, wav=actual)
|
||||
|
||||
source = read_output(args.reference, "wav")
|
||||
report = {
|
||||
"scope": "local x86-64 LiteRT CPU execution of one retained Code2Wav TFLite binary; no hosted-device timing or complete TTS",
|
||||
"status": "component_cpu_numeric_parity_on_one_fixture",
|
||||
"runtime": {"ai_edge_litert": importlib.metadata.version("ai-edge-litert"), "numpy": np.__version__},
|
||||
"host": {"system": platform.system(), "release": platform.release(), "machine": platform.machine(), "processor": cpu_model()},
|
||||
"threads": args.threads,
|
||||
"model_sha256": sha256(args.model),
|
||||
"fixture_sha256": sha256(args.fixture),
|
||||
"reference_sha256": sha256(args.reference),
|
||||
"output_sha256": sha256(args.output),
|
||||
"source_onnx_cpu_vs_litert_cpu": compare(source, actual),
|
||||
"single_invoke_seconds_not_a_profile": elapsed,
|
||||
"limits": [
|
||||
"Local x86-64 CPU execution cannot attest SA8775P or RB3 device CPU behavior.",
|
||||
"LiteRT CPU output is not evidence that the Android GPU delegate is correct.",
|
||||
"One fixed vocoder fixture has no listening-quality tolerance or complete-stream gate.",
|
||||
],
|
||||
}
|
||||
args.report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"status": report["status"], "comparison": report["source_onnx_cpu_vs_litert_cpu"]}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,5 +1,7 @@
|
||||
# 2026-09-24 Qualcomm component expansion
|
||||
|
||||
The [Qwen3-TTS vocoder study](evidence/qwen_tts_vocoder_qualcomm/README.md) reuses a retained fixed-shape Code2Wav export and historical Workbench input to test numerical behavior on SA8775P and other Qualcomm settings. The fresh SA8775P GPU inference produces a saturated waveform; RB3 produces an unsaturated but divergent waveform. Neither is numerically qualified as a usable TTS path. X Elite ONNX CPU output matches the local ONNX Runtime reference on the retained fixture; its profile and DirectML retry remain pending. Other Workbench jobs for RB3 profiling and SA8775P controls are also pending; their submission IDs are retained for completion. No complete on-device TTS stream or model-quality qualification is claimed.
|
||||
The [Qwen3-TTS vocoder study](evidence/qwen_tts_vocoder_qualcomm/README.md) reuses a retained fixed-shape Code2Wav export and historical Workbench input to test numerical behavior on SA8775P and other Qualcomm settings. The fresh SA8775P GPU inference produces a saturated waveform; RB3 produces an unsaturated but divergent waveform. Neither is numerically qualified as a usable TTS path. X Elite ONNX CPU output matches the local ONNX Runtime reference on the retained fixture; its 94-sample component p50/p95 was 6.384/6.644 s with all execution-detail rows on CPU. Its DirectML retry remains pending. Other Workbench jobs for RB3 profiling and SA8775P controls are also pending; their submission IDs are retained for completion. No complete on-device TTS stream or model-quality qualification is claimed.
|
||||
|
||||
A local HX370 x86-64 LiteRT CPU control of the identical SA/RB3 TFLite binary matched the original ONNX CPU waveform within 3.45e-6 relative L2. The SA saturation is therefore not universal to the converted artifact; hosted CPU and GPU precision controls are still needed to isolate its device/runtime cause.
|
||||
|
||||
The [talker decode-step attempt](evidence/qwen_tts_talker_qualcomm/README.md) pins a separate 28-layer source and synthetic cache fixture. Historical FP16 QNN context-binary compiles passed for S24, S25, SA8775P and X Elite, with same-fixture inference pending; RB3 rejected that floating-point-input artifact. Compile success is not a numerical or end-to-end pass.
|
||||
+2
@@ -6,4 +6,6 @@ The deterministic synthetic input dataset `d7dw6pgy2` has SHA256 `0bf538e1b64dfc
|
||||
|
||||
Historical FP16 QNN context-binary [S24](s24/compile_report.json), [S25](s25/compile_report.json), [SA8775P](sa8775p/compile_report.json), and [Snapdragon X Elite](xelite/compile_report.json) compile jobs succeeded. Their same-fixture NPU inference jobs were [submitted](s24/inference_submission.json), [submitted](s25/inference_submission.json), [submitted](sa8775p/inference_submission.json), and [submitted](xelite/inference_submission.json), respectively; none had a terminal result at this snapshot. The [RB3 compile](rb3/compile_report.json) rejected the floating-point input on this device. These are artifact-specific compile dispositions, not full-model support claims.
|
||||
|
||||
The [collector](../../../../experiments/collect_aihub_component.py) retrieves terminal inference outputs without resubmission. The [tensor auditor](../../../../experiments/audit_qwen_tts_talker_qualcomm.py) enforces the source, fixture, compile target, exact device, output shape and finiteness contracts, and reports logits top-1 plus relative error for all 56 new K/V tensors. It does not convert those metrics to a quality pass without an explicit token/audio tolerance. Its 57-output contract was exercised against the pinned CPU reference as a local self-check; that self-check is not device evidence.
|
||||
|
||||
Next: retrieve terminal inference tensors, compare logits and every K/V output against the pinned CPU reference with an explicit tolerance, inspect actual placement, then profile a quality-passing decode step. Full device-local talker/predictor/vocoder streaming and power/memory qualification remain separate gates.
|
||||
+4
-2
@@ -8,10 +8,12 @@ The [S25 historical inference](s25_reference_inference.json) on the same fixture
|
||||
|
||||
The fresh [RB3 inference](rb3/inference_report.json) on the same fixture returned a finite, **unsaturated** waveform ([output](rb3/device_output.npz)). Its [comparison](rb3/inference_comparison.json) against original ONNX CPU was **15.73% relative L2 / 16.06 dB SNR**, still without a listening-quality threshold. Thus the bitwise identical SA and RB3 TFLite target downloads produce materially different outputs on their respective devices; target file identity does not establish runtime numerical parity. The fresh RB3 profile was still pending at this snapshot.
|
||||
|
||||
A separate [local LiteRT 2.2.0 CPU control](local_litert_cpu/report.json) ran that **same pinned TFLite binary** on the WSL x86-64 HX370 host with four interpreter threads. Its [waveform](local_litert_cpu/device_output.npz) was finite and unsaturated, within **3.45e-6 relative L2 / 109.23 dB SNR** of the original ONNX Runtime CPU output. The [reproducer](../../../../experiments/probe_qwen_tts_vocoder_litert_cpu.py) checks binary/fixture/reference hashes and I/O contracts. This narrows the SA saturation to hosted runtime/device behavior rather than a universally bad TFLite conversion. The one local invoke is not a comparable performance profile; it does not prove either embedded device's CPU route or identify the specific delegate operation at fault.
|
||||
|
||||
The re-fetched *historical* [SA8775P profile](sa8775p/historical_profile_report.json) has 100 component samples, nearest-rank p50/p95 **1.751/1.874 s**, and 403 CPU plus 335 GPU execution-detail rows. The *historical* [RB3 profile](rb3/historical_profile_report.json) has 100 samples, p50/p95 **3.143/3.189 s**, also with 403 CPU plus 335 GPU rows. Their previously quoted 1.701/3.107 s estimated-time fields are sample minima, not medians. Both profiles exclude talker/predictor, stage handoff, streaming and full-request time.
|
||||
|
||||
The X Elite [ONNX CPU inference](xelite/cpu/inference_report.json) succeeded on the pinned fixture. Its [waveform comparison](xelite/cpu/inference_comparison.json) to the original local ONNX Runtime CPU output was **8.09e-7 relative L2 / 121.84 dB SNR**, with no saturation. This is near-source component parity on one fixture; the CPU profile and listening-quality gate remain open. The first X Elite [GPU inference](xelite/gpu/inference_report.json) and [profile](xelite/gpu/profile_report.json) jobs failed because Workbench rejects specifying both compute unit and ONNX execution provider; the [DirectML retry](xelite/directml/inference_submission.json) specifies DirectML alone. The [X Elite ONNX compile](xelite/compile_report.json) passed after the older TFLite target was rejected on Windows ARM.
|
||||
The X Elite [ONNX CPU inference](xelite/cpu/inference_report.json) succeeded on the pinned fixture. Its [waveform comparison](xelite/cpu/inference_comparison.json) to the original local ONNX Runtime CPU output was **8.09e-7 relative L2 / 121.84 dB SNR**, with no saturation. The fresh [CPU profile](xelite/cpu/profile_report.json) returned **94** component samples, nearest-rank p50/p95 **6.384/6.644 s**, with all **634** execution-detail rows on CPU; the service's 5.917 s estimated-time field is the sample minimum. The [full audit](xelite/cpu/audit_report.json) pins source, fixture, target, waveform, placement and percentiles. This is near-source component parity on one fixture, without a listening-quality or complete-stream gate. The first X Elite [GPU inference](xelite/gpu/inference_report.json) and [profile](xelite/gpu/profile_report.json) jobs failed because Workbench rejects specifying both compute unit and ONNX execution provider; the [DirectML retry](xelite/directml/inference_submission.json) specifies DirectML alone. The [X Elite ONNX compile](xelite/compile_report.json) passed after the older TFLite target was rejected on Windows ARM.
|
||||
|
||||
The fresh [RB3 profile](rb3/profile_submission.json), [SA CPU](sa8775p/cpu_control/inference_submission.json) and [FP32-preserving GPU](sa8775p/gpu_fp32/inference_submission.json) controls, X Elite [DirectML retry](xelite/directml/inference_submission.json) and [CPU profile](xelite/cpu/profile_submission.json) remained pending in this snapshot. Pending jobs have no numerical or latency conclusion. The [collector](../../../../experiments/collect_aihub_component.py) retrieves their terminal reports without resubmitting jobs, and the [audit script](../../../../experiments/audit_qwen_tts_vocoder_qualcomm.py) checks source/fixture hashes, device and model IDs, actual placement, waveform error, saturation and sample percentiles.
|
||||
The fresh [RB3 profile](rb3/profile_submission.json), [SA CPU](sa8775p/cpu_control/inference_submission.json) and [FP32-preserving GPU](sa8775p/gpu_fp32/inference_submission.json) controls, and X Elite [DirectML retry](xelite/directml/inference_submission.json) remained pending in this snapshot. Pending jobs have no numerical or latency conclusion. The [collector](../../../../experiments/collect_aihub_component.py) retrieves their terminal reports without resubmitting jobs, and the [audit script](../../../../experiments/audit_qwen_tts_vocoder_qualcomm.py) checks source/fixture hashes, device and model IDs, actual placement, waveform error, saturation and sample percentiles.
|
||||
|
||||
This is one fixed vocoder component with a retained input, not complete Qwen3-TTS. The next gates are a quality-passing SA/RB3/X Elite target, talker and code predictor numerical checks, device-resident handoff/co-residency, playable streaming, cancellation, memory admission and sustained power/thermal profiling.
|
||||
BIN
Binary file not shown.
+34
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"scope": "local x86-64 LiteRT CPU execution of one retained Code2Wav TFLite binary; no hosted-device timing or complete TTS",
|
||||
"status": "component_cpu_numeric_parity_on_one_fixture",
|
||||
"runtime": {
|
||||
"ai_edge_litert": "2.2.0",
|
||||
"numpy": "2.5.3"
|
||||
},
|
||||
"host": {
|
||||
"system": "Linux",
|
||||
"release": "6.18.33.2-microsoft-standard-WSL2",
|
||||
"machine": "x86_64",
|
||||
"processor": "AMD Ryzen AI 9 HX 370 w/ Radeon 890M"
|
||||
},
|
||||
"threads": 4,
|
||||
"model_sha256": "29826da6ecc1ec2efc0e3589c3cebb42b20d7e20c0975a037d9f51625d6638cd",
|
||||
"fixture_sha256": "dfca9e8d2a724a568a31b6cd18ff6a52fdc205fd34e13cbf1cd75f0f7937e82d",
|
||||
"reference_sha256": "8de78b94b59f01e49852f7706adbd2508a9976b8e2705cfd3b94d07bb3606cb1",
|
||||
"output_sha256": "c19524684abaf5da5c5f2c539a536ddba3ecc5e094c252e1dadc2804f0b482c7",
|
||||
"source_onnx_cpu_vs_litert_cpu": {
|
||||
"relative_l2": 3.454229291440569e-06,
|
||||
"snr_db": 109.23297674763376,
|
||||
"max_abs": 5.692243576049805e-06,
|
||||
"cosine": 0.9999999999940402,
|
||||
"reference_rms": 0.08373439205485074,
|
||||
"output_rms": 0.08373440156017416,
|
||||
"output_saturated_fraction_abs_ge_0_999": 0.0
|
||||
},
|
||||
"single_invoke_seconds_not_a_profile": 1.6104475790052675,
|
||||
"limits": [
|
||||
"Local x86-64 CPU execution cannot attest SA8775P or RB3 device CPU behavior.",
|
||||
"LiteRT CPU output is not evidence that the Android GPU delegate is correct.",
|
||||
"One fixed vocoder fixture has no listening-quality tolerance or complete-stream gate."
|
||||
]
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"scope": "one Qwen3-TTS code2wav 48,000-sample component on one retained fixture; no complete TTS stream",
|
||||
"status": "component_executed_quality_unqualified",
|
||||
"device": {
|
||||
"name": "Snapdragon X Elite CRD",
|
||||
"os": "11",
|
||||
"attributes": [
|
||||
"os:windows",
|
||||
"format:compute",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-windows",
|
||||
"chipset:qualcomm-snapdragon-x-elite",
|
||||
"chipset:sc8380xp",
|
||||
"hexagon:v73",
|
||||
"soc-model:60",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"compile_job_id": "j5qlyz8op",
|
||||
"inference_job_id": "j5wl78nmp",
|
||||
"profile_job_id": "jg9zmke8p",
|
||||
"target_model_id": "mnz079ezm",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"s25_prior_inference_job_id": "jpx428ojg",
|
||||
"requested_run_options": "--compute_unit cpu",
|
||||
"files": {
|
||||
"source_onnx": {
|
||||
"bytes": 423655047,
|
||||
"sha256": "7c07666229c6d404894132e22f784525e3212a49efad55ddd532eda5286c2b2b"
|
||||
},
|
||||
"fixture": {
|
||||
"bytes": 129721,
|
||||
"sha256": "dfca9e8d2a724a568a31b6cd18ff6a52fdc205fd34e13cbf1cd75f0f7937e82d"
|
||||
},
|
||||
"cpu_output": {
|
||||
"bytes": 179942,
|
||||
"sha256": "8de78b94b59f01e49852f7706adbd2508a9976b8e2705cfd3b94d07bb3606cb1"
|
||||
},
|
||||
"s25_output": {
|
||||
"bytes": 115235,
|
||||
"sha256": "4a26486d5e84a6e22d16f9b26f995b8fe645c29bbd70b103615ec35a0fcb10a0"
|
||||
},
|
||||
"device_output": {
|
||||
"bytes": 179943,
|
||||
"sha256": "de7e4605bc6508c19c22ca3e3f5d2d67c1002908aa1e3cebfe7f40bd040f304a"
|
||||
}
|
||||
},
|
||||
"cpu_source_vs_device": {
|
||||
"relative_l2": 8.086912399814565e-07,
|
||||
"snr_db": 121.84434522578867,
|
||||
"max_abs": 9.5367431640625e-07,
|
||||
"cosine": 0.9999999999996769,
|
||||
"reference_rms": 0.08373439205485074,
|
||||
"output_rms": 0.0837343848397309,
|
||||
"output_saturated_fraction_abs_ge_0_999": 0.0
|
||||
},
|
||||
"s25_gpu_vs_device": {
|
||||
"relative_l2": 0.04236831072350974,
|
||||
"snr_db": 27.45917703014474,
|
||||
"max_abs": 0.10591856762766838,
|
||||
"cosine": 0.9991021841139998,
|
||||
"reference_rms": 0.08376787228276501,
|
||||
"output_rms": 0.0837343848397309,
|
||||
"output_saturated_fraction_abs_ge_0_999": 0.0
|
||||
},
|
||||
"profile": {
|
||||
"sample_count": 94,
|
||||
"sample_min_us": 5916879,
|
||||
"nearest_rank_p50_us": 6384115,
|
||||
"nearest_rank_p95_us": 6644231,
|
||||
"reported_estimated_inference_time_us": 5916879,
|
||||
"reported_peak_memory_bytes": 1340260352,
|
||||
"compute_unit_row_counts": {
|
||||
"CPU": 634
|
||||
}
|
||||
},
|
||||
"limits": [
|
||||
"Historical source export does not attest its exact checkpoint revision.",
|
||||
"This fixed component waveform has no listening or task-quality tolerance.",
|
||||
"Hosted component profiling excludes talker, predictor, stage handoff, streaming, admission and sustained power/thermal behavior."
|
||||
]
|
||||
}
|
||||
+4627
File diff suppressed because it is too large.
Load diff
@@ -12,7 +12,9 @@ On a hosted [Galaxy S24 Spark attention-layer experiment](../../../e2e_expansion
|
||||
|
||||
The same W8A16 layer also ran on hosted [Snapdragon X Elite CRD](../../../e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md) and [SA8775P ADP](../../../e2e_expansion_20260923/evidence/spark_sa8775_full_attention/README.md) NPUs, with pinned-fixture outputs identical to S24/S25; the 100-sample component p50/p95 was 1.433/1.606 ms and 1.837/2.079 ms, respectively. On [RB3 Gen 2 / QCS6490](../../../e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md), that W8A16 graph failed QNN DLC loading even after target-specific recompilation. An FP32 ONNX CPU layer then passed with source-relative L2 below 9e-7, while a calibrated W8A8 NPU layer executed but had 20.9% hidden-state relative L2 and remains numerically unqualified. These hosted component outcomes do not establish full generation or comparable device speed.
|
||||
|
||||
A [Qwen3-TTS Code2Wav follow-up](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) recovered source and target artifacts, a retained fixture, and raw historical profiles. On SA8775P, fresh GPU-requested inference saturated 96.63% of waveform samples and differed from original ONNX CPU by 11.832 relative L2; this tested vocoder path is numerically unqualified. Fresh RB3 inference was unsaturated but differed from CPU by 15.73% relative L2 (16.06 dB SNR), also without a listening-quality gate. Historical SA8775P/RB3 profile p50/p95 values were 1.751/1.874 s and 3.143/3.189 s, respectively, with both CPU and GPU rows; earlier estimated-time fields were minima. X Elite ONNX CPU vocoder inference passed on the same fixture with 8.09e-7 relative L2 versus local ONNX CPU; its profile and DirectML retry were pending. None is a complete TTS stream.
|
||||
A [Qwen3-TTS Code2Wav follow-up](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) recovered source and target artifacts, a retained fixture, and raw historical profiles. On SA8775P, fresh GPU-requested inference saturated 96.63% of waveform samples and differed from original ONNX CPU by 11.832 relative L2; this tested vocoder path is numerically unqualified. Fresh RB3 inference was unsaturated but differed from CPU by 15.73% relative L2 (16.06 dB SNR), also without a listening-quality gate. Historical SA8775P/RB3 profile p50/p95 values were 1.751/1.874 s and 3.143/3.189 s, respectively, with both CPU and GPU rows; earlier estimated-time fields were minima. X Elite ONNX CPU vocoder inference passed on the same fixture with 8.09e-7 relative L2 versus local ONNX CPU; its 94-sample component p50/p95 was 6.384/6.644 s with all execution-detail rows on CPU. DirectML retry was pending. None is a complete TTS stream.
|
||||
|
||||
The identical SA/RB3 TFLite binary matched original ONNX CPU within 3.45e-6 relative L2 in a separate local HX370 x86-64 LiteRT CPU control. This narrows the SA waveform saturation to hosted device/runtime behavior, without proving either embedded CPU route or a specific GPU-delegate fault.
|
||||
|
||||
A separate [Qwen3-TTS talker decode-step attempt](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) recovered a 28-layer ONNX export and synthetic fixed-cache fixture. Historical FP16 QNN context-binary compiles passed for S24, S25, SA8775P and X Elite, but the same-fixture inference jobs were pending. An RB3 compile rejected floating-point input. No talker runtime or numerical pass is implied by those compile outcomes.
|
||||
|
||||
@@ -81,7 +83,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | InternVLA-A1 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Spark-X2.5 | COMPONENT_NPU_INFERENCE_PROFILED; NOT_E2E_PROFILEABLE | A real Spark-X2.5-1.7B one-layer W8A16 QDQ attention artifact, compiled as QNN DLC, ran on the exact hosted Snapdragon X Elite CRD (Windows 11 ARM64). Inference returned finite hidden/new-K/new-V tensors bitwise identical to the prior S24/S25 NPU outputs on the same fixture; the profile attributed all 73 execution-detail rows to NPU. Across 100 component samples, nearest-rank p50/p95 was 1.433/1.606 ms; the 1.346 ms estimated-time field is the minimum. Hidden/K/V relative L2 versus original FP32 ONNX CPU was 1.351%/0.377%/0.650%, without a next-token quality tolerance. There is no complete 28-layer prefill/decode/KV-ring/sampling loop, resident Omni controller or complete-request timing. | Qualify a complete X Elite resident artifact/controller with state, token quality, memory admission/cancel and sustained power; [X Elite NPU raw evidence](../../../e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md). |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_ONNX_CPU_VOCODER_PARITY_PASS; NOT_E2E_PROFILEABLE | The fresh ONNX vocoder inference on the exact Windows 11 ARM64 X Elite CRD requested CPU and returned an unsaturated waveform on the pinned fixture, matching local ONNX Runtime CPU with 8.09e-7 relative L2 and 121.84 dB SNR. This is one component output, without a listening-quality gate. CPU profile and corrected DirectML inference/profile jobs were pending; the initial DirectML submissions failed because mutually exclusive run options were specified. Historical QNN talker profiling lacks a same-fixture numerical or full-pipeline gate. The older TFLite vocoder target was rejected on Windows ARM; no complete stream exists. | Audit CPU profile and DirectML retry, qualify talker/predictor and complete local streaming, memory and sustained behavior; [vocoder evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md). |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_ONNX_CPU_VOCODER_PARITY_PROFILED; NOT_E2E_PROFILEABLE | The fresh ONNX vocoder inference on the exact Windows 11 ARM64 X Elite CRD requested CPU and returned an unsaturated waveform on the pinned fixture, matching local ONNX Runtime CPU with 8.09e-7 relative L2 and 121.84 dB SNR. Its 94-sample component profile had nearest-rank p50/p95 6.384/6.644 s and all 634 execution-detail rows on CPU; 5.917 s estimated time is the sample minimum. This is one component, without a listening-quality gate. Corrected DirectML inference/profile jobs were pending; the initial DirectML submissions failed because mutually exclusive run options were specified. Historical QNN talker profiling lacks a same-fixture numerical or full-pipeline gate. The older TFLite vocoder target was rejected on Windows ARM; no complete stream exists. | Audit DirectML retry, qualify talker/predictor and complete local streaming, memory and sustained behavior; [vocoder evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md). |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
|
||||
Reference in new issue
Block a user