Audit Qualcomm Qwen3-TTS vocoder parity and profile
This commit is contained in:
1 parent
bd39e903e1
commit
152d67ab86
33 files changed
+16758
-8
No files matched your search
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit a hosted Qwen3-TTS vocoder component against a pinned ONNX CPU run."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
SOURCE_SHA256 = "7c07666229c6d404894132e22f784525e3212a49efad55ddd532eda5286c2b2b"
|
||||
FIXTURE_SHA256 = "dfca9e8d2a724a568a31b6cd18ff6a52fdc205fd34e13cbf1cd75f0f7937e82d"
|
||||
CPU_OUTPUT_SHA256 = "8de78b94b59f01e49852f7706adbd2508a9976b8e2705cfd3b94d07bb3606cb1"
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
with path.open("rb") as source:
|
||||
return hashlib.file_digest(source, "sha256").hexdigest()
|
||||
|
||||
|
||||
def read_output(path: Path, name: str) -> np.ndarray:
|
||||
with np.load(path, allow_pickle=False) as data:
|
||||
if data.files != [name]:
|
||||
raise ValueError(f"{path}: expected only {name}, got {data.files}")
|
||||
output = np.asarray(data[name])
|
||||
if output.shape != (1, 48000) or output.dtype != np.float32 or not np.isfinite(output).all():
|
||||
raise ValueError(f"{path}: violates finite FP32 48,000-sample WAV contract")
|
||||
return output
|
||||
|
||||
|
||||
def compare(reference: np.ndarray, output: np.ndarray) -> dict:
|
||||
a = reference.astype(np.float64).ravel()
|
||||
b = output.astype(np.float64).ravel()
|
||||
error_norm = float(np.linalg.norm(a - b))
|
||||
reference_norm = float(np.linalg.norm(a))
|
||||
output_norm = float(np.linalg.norm(b))
|
||||
if reference_norm == 0 or output_norm == 0:
|
||||
raise ValueError("silent reference/output cannot define waveform similarity")
|
||||
return {
|
||||
"relative_l2": error_norm / reference_norm,
|
||||
"snr_db": 20 * math.log10(reference_norm / error_norm) if error_norm else None,
|
||||
"max_abs": float(np.max(np.abs(a - b))),
|
||||
"cosine": float(np.dot(a, b) / (reference_norm * output_norm)),
|
||||
"reference_rms": float(np.sqrt(np.mean(a * a))),
|
||||
"output_rms": float(np.sqrt(np.mean(b * b))),
|
||||
"output_saturated_fraction_abs_ge_0_999": float(np.mean(np.abs(b) >= 0.999)),
|
||||
}
|
||||
|
||||
|
||||
def nearest_rank(values: list[int], fraction: float) -> int:
|
||||
ordered = sorted(values)
|
||||
return ordered[math.ceil(fraction * len(ordered)) - 1]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--device-name", required=True)
|
||||
parser.add_argument("--compile-options", required=True)
|
||||
parser.add_argument("--run-options", required=True)
|
||||
for name in (
|
||||
"source-onnx", "fixture", "cpu-output", "s25-output", "s25-inference-report", "compile-report",
|
||||
"inference-report", "profile-report", "device-output", "output-report",
|
||||
):
|
||||
parser.add_argument(f"--{name}", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
compile_report = json.loads(args.compile_report.read_text(encoding="utf-8"))
|
||||
inference = json.loads(args.inference_report.read_text(encoding="utf-8"))
|
||||
profile = json.loads(args.profile_report.read_text(encoding="utf-8"))
|
||||
s25_inference = json.loads(args.s25_inference_report.read_text(encoding="utf-8"))
|
||||
if any(report.get("status") != "SUCCESS" for report in (compile_report, inference, profile)):
|
||||
raise RuntimeError("compile, inference and profile must all succeed")
|
||||
if compile_report.get("source_model_id") != "mqyer339n" or compile_report.get("options") != args.compile_options:
|
||||
raise RuntimeError("compile did not use pinned Qwen3-TTS vocoder source/options")
|
||||
target_id = compile_report["target_model_id"]
|
||||
if inference.get("model_id") != target_id or profile.get("model_id") != target_id:
|
||||
raise RuntimeError("inference/profile used a different target artifact")
|
||||
if inference.get("input_dataset_id") != "d7m80jl32":
|
||||
raise RuntimeError("inference did not use the retained vocoder fixture")
|
||||
if (s25_inference.get("status") != "SUCCESS" or s25_inference.get("model_id") != "mng7kwxon"
|
||||
or s25_inference.get("input_dataset_id") != inference["input_dataset_id"]
|
||||
or s25_inference.get("device", {}).get("name") != "Samsung Galaxy S25"):
|
||||
raise RuntimeError("historical S25 output lacks matching model/device/dataset provenance")
|
||||
for report in (compile_report, inference, profile):
|
||||
if report.get("device", {}).get("name") != args.device_name:
|
||||
raise RuntimeError("job did not use the exact requested device")
|
||||
if inference.get("options") != args.run_options or profile.get("options") != args.run_options:
|
||||
raise RuntimeError("inference/profile requested different compute options")
|
||||
files = {
|
||||
name: {"bytes": path.stat().st_size, "sha256": sha256(path)}
|
||||
for name, path in {
|
||||
"source_onnx": args.source_onnx,
|
||||
"fixture": args.fixture,
|
||||
"cpu_output": args.cpu_output,
|
||||
"s25_output": args.s25_output,
|
||||
"device_output": args.device_output,
|
||||
}.items()
|
||||
}
|
||||
for name, expected in {
|
||||
"source_onnx": SOURCE_SHA256,
|
||||
"fixture": FIXTURE_SHA256,
|
||||
"cpu_output": CPU_OUTPUT_SHA256,
|
||||
}.items():
|
||||
if files[name]["sha256"] != expected:
|
||||
raise RuntimeError(f"{name} differs from the pinned source/fixture/reference")
|
||||
with np.load(args.fixture, allow_pickle=False) as data:
|
||||
if data.files != ["quantized"] or data["quantized"].shape != (1, 512, 97):
|
||||
raise RuntimeError("fixture input contract differs")
|
||||
cpu = read_output(args.cpu_output, "wav")
|
||||
s25 = read_output(args.s25_output, "output_0__0")
|
||||
output = read_output(args.device_output, "output_0__0")
|
||||
execution = profile["profile"]
|
||||
times_us = execution["execution_summary"]["all_inference_times"]
|
||||
if not times_us or any(not isinstance(value, int) or value <= 0 for value in times_us):
|
||||
raise RuntimeError("missing or invalid device samples")
|
||||
units = Counter(row.get("compute_unit") for row in execution["execution_detail"])
|
||||
if not units:
|
||||
raise RuntimeError("no compute-unit placement detail")
|
||||
report = {
|
||||
"scope": "one Qwen3-TTS code2wav 48,000-sample component on one retained fixture; no complete TTS stream",
|
||||
"status": "component_executed_quality_unqualified",
|
||||
"device": profile["device"],
|
||||
"compile_job_id": compile_report["job_id"],
|
||||
"inference_job_id": inference["job_id"],
|
||||
"profile_job_id": profile["job_id"],
|
||||
"target_model_id": target_id,
|
||||
"input_dataset_id": inference["input_dataset_id"],
|
||||
"s25_prior_inference_job_id": s25_inference["job_id"],
|
||||
"requested_run_options": args.run_options,
|
||||
"files": files,
|
||||
"cpu_source_vs_device": compare(cpu, output),
|
||||
"s25_gpu_vs_device": compare(s25, output),
|
||||
"profile": {
|
||||
"sample_count": len(times_us),
|
||||
"sample_min_us": min(times_us),
|
||||
"nearest_rank_p50_us": nearest_rank(times_us, 0.50),
|
||||
"nearest_rank_p95_us": nearest_rank(times_us, 0.95),
|
||||
"reported_estimated_inference_time_us": execution["execution_summary"]["estimated_inference_time"],
|
||||
"reported_peak_memory_bytes": execution["execution_summary"]["estimated_inference_peak_memory"],
|
||||
"compute_unit_row_counts": dict(units),
|
||||
},
|
||||
"limits": [
|
||||
"Historical source export does not attest its exact checkpoint revision.",
|
||||
"This fixed component waveform has no listening or task-quality tolerance.",
|
||||
"Hosted component profiling excludes talker, predictor, stage handoff, streaming, admission and sustained power/thermal behavior.",
|
||||
],
|
||||
}
|
||||
args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"status": report["status"], "profile": report["profile"]}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+3
-2
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Preserve raw Workbench Spark-layer jobs submitted in an evidence directory.
|
||||
"""Preserve raw Workbench component jobs submitted in an evidence directory.
|
||||
|
||||
Requires qai-hub authentication through the local client configuration. The
|
||||
submission JSON files identify existing jobs; this script never resubmits them.
|
||||
@@ -70,10 +70,11 @@ def collect(evidence_dir: Path, kind: str, client: hub.Client) -> str:
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("evidence_dirs", nargs="+", type=Path)
|
||||
parser.add_argument("--kinds", nargs="+", choices=("inference", "profile"), default=("inference", "profile"))
|
||||
args = parser.parse_args()
|
||||
client = hub.Client()
|
||||
for directory in args.evidence_dirs:
|
||||
for kind in ("inference", "profile"):
|
||||
for kind in args.kinds:
|
||||
print(f"{directory.name}: {collect(directory, kind, client)}")
|
||||
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ Four routes were tested on exact device **Dragonwing RB3 Gen 2 Vision Kit**, usi
|
||||
| Compile original FP32 source to ONNX `mmxj02ljq`, request CPU | [Compile](compile_cpu_report.json) `jglyxowj5`, [inference](cpu/inference_report.json) `jgolrodqg` and [profile](cpu/profile_report.json) `jpvldemk5` passed. All 56 execution-detail rows were CPU. | [Output](cpu/device_output.npz) hidden/K/V relative L2 versus original ONNX Runtime 1.29 CPU was 8.73e-7/2.98e-7/3.37e-7. [Audit](cpu/audit_report.json): 100 samples, nearest-rank p50/p95 19.304/21.113 ms (minimum 19.104 ms). This is explicitly CPU, not NPU fallback masquerading as acceleration. |
|
||||
| Compile original source with calibration dataset `d7x8e3d59` using `--quantize_full_type int8 --quantize_io` to QNN DLC `mqkp10ezm`, request NPU | [Compile](compile_w8a8_report.json) `jpyoxq885`, [inference](w8a8/inference_report.json) `jgzlm8yo5` and [profile](w8a8/profile_report.json) `j5wl78z3p` passed. All 61 execution-detail rows were NPU. | [Output](w8a8/device_output.npz) hidden/K/V relative L2 versus FP32 CPU was **20.914%/1.702%/4.413%**. [Audit](w8a8/audit_report.json): 100 samples, p50/p95 5.610/5.761 ms (minimum 5.089 ms). This precision is **numerically unqualified**; no next-token/task tolerance is established. |
|
||||
|
||||
All successful jobs used the same fixture dataset `d7m83gjy2` and returned finite FP32 hidden/new-key/new-value tensors. The service `estimated_inference_time` equals each sample minimum, not the median. The [CPU](cpu/audit_report.json) and [W8A8 NPU](w8a8/audit_report.json) audits pin source and fixture hashes, exact job/device/model IDs, output hashes, placement rows and full precision differences. [Submission records](compile_w8a8_submission.json), [initial failed submissions](inference_submission.json), [RB3 W8A16 compile submission](compile_rb3_submission.json), and [CPU compile submission](compile_cpu_submission.json) preserve attempted settings. The scripts [collect existing jobs](../../../../experiments/collect_spark_aihub_component.py) and [audit fallback variants](../../../../experiments/audit_spark_qualcomm_fallback.py) without embedded credentials.
|
||||
All successful jobs used the same fixture dataset `d7m83gjy2` and returned finite FP32 hidden/new-key/new-value tensors. The service `estimated_inference_time` equals each sample minimum, not the median. The [CPU](cpu/audit_report.json) and [W8A8 NPU](w8a8/audit_report.json) audits pin source and fixture hashes, exact job/device/model IDs, output hashes, placement rows and full precision differences. [Submission records](compile_w8a8_submission.json), [initial failed submissions](inference_submission.json), [RB3 W8A16 compile submission](compile_rb3_submission.json), and [CPU compile submission](compile_cpu_submission.json) preserve attempted settings. The scripts [collect existing jobs](../../../../experiments/collect_aihub_component.py) and [audit fallback variants](../../../../experiments/audit_spark_qualcomm_fallback.py) without embedded credentials.
|
||||
|
||||
The CPU and NPU numbers are different precisions and outputs, not a qualified speedup comparison. Neither job includes 27 other decoder layers, prefill, continuous decode, KV/ring, sampling, a resident Omni controller, complete-request timing or sustained power/thermal behavior. Workbench is a test facility, not a deployment dependency.
|
||||
|
||||
|
||||
+1
-1
@@ -8,6 +8,6 @@ The [three outputs](device_output.npz) were finite FP32 hidden/new-key/new-value
|
||||
|
||||
All 73 execution-detail rows in the profile were attributed to NPU. The 100 component samples had nearest-rank p50/p95 **1.837/2.079 ms** (minimum 1.749 ms); the service's `estimated_inference_time` was that minimum, not the median. Reported component inference peak memory was 90,877,952 bytes. These hosted component measurements exclude prefill, 27 other decoder layers, sampling, handoff, loading and a resident KV/ring/controller. They are not paired speed comparisons with other devices or OS builds.
|
||||
|
||||
The [audit script](../../../../experiments/audit_spark_qualcomm_attention.py) reproduces the provenance and numerical checks using pinned source artifact paths in the S24 record. The [collector](../../../../experiments/collect_spark_aihub_component.py) retrieves these existing jobs via the local client configuration; neither script contains credentials.
|
||||
The [audit script](../../../../experiments/audit_spark_qualcomm_attention.py) reproduces the provenance and numerical checks using pinned source artifact paths in the S24 record. The [collector](../../../../experiments/collect_aihub_component.py) retrieves these existing jobs via the local client configuration; neither script contains credentials.
|
||||
|
||||
**Matrix status: component NPU inference/profiled; NOT E2E.** Full qualification needs a complete device-resident Spark artifact and Omni backend with prefill/decode/KV-ring/sampling, state, token quality, cancellation and memory admission, followed by sustained power and thermal measurement.
|
||||
+1
-1
@@ -8,6 +8,6 @@ The [three outputs](device_output.npz) were finite FP32 hidden/new-key/new-value
|
||||
|
||||
All 73 execution-detail rows in the device profile were attributed to NPU. The 100 component inference-time samples had nearest-rank p50/p95 **1.433/1.606 ms** (minimum 1.346 ms). The service's `estimated_inference_time` was 1.346 ms, matching the minimum rather than the median. The reported component inference peak memory was 18,423,808 bytes. These are hosted component-job measurements; they exclude prefill, 27 other decoder layers, sampling, transfer, loading and a resident KV/ring/controller. Hardware, OS and hosted power state differ from the mobile runs, so these are not paired speed comparisons.
|
||||
|
||||
Re-audit with `benchmarks/edge_harness/experiments/audit_spark_qualcomm_attention.py` and the [pinned source artifact paths](../spark_s24_full_attention/README.md). `benchmarks/edge_harness/experiments/collect_spark_aihub_component.py` retrieves the existing jobs using the local AI Hub client configuration. Neither script contains credentials.
|
||||
Re-audit with `benchmarks/edge_harness/experiments/audit_spark_qualcomm_attention.py` and the [pinned source artifact paths](../spark_s24_full_attention/README.md). `benchmarks/edge_harness/experiments/collect_aihub_component.py` retrieves the existing jobs using the local AI Hub client configuration. Neither script contains credentials.
|
||||
|
||||
**Matrix status: component NPU inference/profiled; NOT E2E.** Full qualification still needs an X Elite machine running the complete 28-layer Spark model through a resident Omni backend with prefill/decode/KV-ring/sampling, state and cancellation, token-quality checks, memory admission and sustained power/thermal measurements. Workbench is a test facility, not that deployment runtime.
|
||||
@@ -0,0 +1,3 @@
|
||||
# 2026-09-24 Qualcomm component expansion
|
||||
|
||||
The [Qwen3-TTS vocoder study](evidence/qwen_tts_vocoder_qualcomm/README.md) reuses a retained fixed-shape Code2Wav export and historical Workbench input to test numerical behavior on SA8775P and other Qualcomm settings. The fresh SA8775P GPU inference produces a saturated waveform, so its historical profile must not be interpreted as a usable TTS path. Workbench jobs for RB3, X Elite and SA8775P controls were still pending when this snapshot was written; their submission IDs are retained for completion. No complete on-device TTS stream or model-quality qualification is claimed.
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
# Qwen3-TTS Code2Wav component across Qualcomm devices
|
||||
|
||||
**The retained Qwen3-TTS Code2Wav export does not yet have a numerically qualified SA8775P GPU path.** A fresh inference on the exact hosted SA8775P ADP returned a severely saturated waveform on the same input for which original ONNX Runtime CPU and prior S25 GPU outputs remained unsaturated. Other submitted device jobs were pending at this evidence snapshot; they are not counted as passes.
|
||||
|
||||
The historical Workbench source model `mqyer339n` is a 423,655,047-byte ONNX graph with one FP32 input `[1,512,97]` and one 48,000-sample output. The [manifest](artifact_manifest.json) pins its SHA256, the retained Workbench fixture dataset `d7m80jl32`, a local ONNX Runtime 1.29 CPU output, compiled target downloads, and the prior [S25 GPU output](s25_reference_output.npz). The source export script identifies the Qwen3-TTS 0.6B CustomVoice model family, but did not attest the exact checkpoint revision; these results must not be promoted to a revision-qualified release claim. Large source/target artifacts and the fixture remain outside Git under `/home/zhout/project/edge_infer/models/`.
|
||||
|
||||
The [S25 historical inference](s25_reference_inference.json) on the same fixture differed from local FP32 ONNX CPU by **4.24% relative L2 / 27.46 dB SNR** ([comparison](s25_cpu_comparison.json)); that waveform has no listening-quality tolerance. A fresh [SA8775P inference](sa8775p/inference_report.json) used historical compiled TFLite model `mnze95don` and requested GPU. Its [output](sa8775p/device_output.npz) had **96.63%** of samples at absolute amplitude at least 0.999, RMS **0.989** versus CPU **0.0837**, relative L2 **11.832**, and SNR **−21.46 dB**. The fresh [profile](sa8775p/profile_report.json) attributed 403 execution-detail rows to CPU and 335 to GPU; 100 component samples had nearest-rank p50/p95 **1.776/1.880 s** (minimum 1.675 s). The [audit](sa8775p/audit_report.json) pins all output hashes, waveform differences and placement. This is measured execution of a **numerically failed** output, not a usable TTS performance result. The [SA8775P compile record](sa8775p/compile_report.json) pins the artifact and source IDs.
|
||||
|
||||
The re-fetched *historical* [SA8775P profile](sa8775p/historical_profile_report.json) has 100 component samples, nearest-rank p50/p95 **1.751/1.874 s**, and 403 CPU plus 335 GPU execution-detail rows. The *historical* [RB3 profile](rb3/historical_profile_report.json) has 100 samples, p50/p95 **3.143/3.189 s**, also with 403 CPU plus 335 GPU rows. Their previously quoted 1.701/3.107 s estimated-time fields are sample minima, not medians. Both profiles exclude talker/predictor, stage handoff, streaming and full-request time. The SA and RB3 TFLite target downloads are bitwise identical, but that does not establish matching runtime outputs or quality.
|
||||
|
||||
Current [RB3](rb3/inference_submission.json) fresh job submissions, [SA CPU](sa8775p/cpu_control/inference_submission.json) and [FP32-preserving GPU](sa8775p/gpu_fp32/inference_submission.json) controls, and X Elite [DirectML GPU](xelite/gpu/inference_submission.json) and [CPU](xelite/cpu/inference_submission.json) submissions remain pending in this snapshot. The [X Elite ONNX compile](xelite/compile_report.json) passed after the older TFLite target was rejected on Windows ARM. Pending jobs have no numerical or latency conclusion. The [collector](../../../../experiments/collect_aihub_component.py) retrieves their terminal reports without resubmitting jobs, and the [audit script](../../../../experiments/audit_qwen_tts_vocoder_qualcomm.py) checks source/fixture hashes, device and model IDs, actual placement, waveform error, saturation and sample percentiles.
|
||||
|
||||
This is one fixed vocoder component with a retained input, not complete Qwen3-TTS. The next gates are a quality-passing SA/RB3/X Elite target, talker and code predictor numerical checks, device-resident handoff/co-residency, playable streaming, cancellation, memory admission and sustained power/thermal profiling.
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"source_model_ids": {
|
||||
"talker": "mq3xre4rq",
|
||||
"vocoder": "mqyer339n"
|
||||
},
|
||||
"fixture_dataset_id": "d7m80jl32",
|
||||
"reference_inference_job_id": "jpx428ojg",
|
||||
"checkpoint_revision": "not attested by historical export; component provenance only",
|
||||
"onnxruntime_cpu_version": "1.29.0",
|
||||
"files": {
|
||||
"talker_source_onnx": {
|
||||
"path": "/home/zhout/project/edge_infer/models/qwen3tts_talker_step2_L256_20260924.onnx",
|
||||
"bytes": 1776377476,
|
||||
"sha256": "41feb08517678e94d9301e992d0df610fc60e7c9faba5600703298c86554fd22"
|
||||
},
|
||||
"vocoder_source_onnx": {
|
||||
"path": "/home/zhout/project/edge_infer/models/qwen3tts_code2wav_c25_20260924.onnx",
|
||||
"bytes": 423655047,
|
||||
"sha256": "7c07666229c6d404894132e22f784525e3212a49efad55ddd532eda5286c2b2b"
|
||||
},
|
||||
"vocoder_fixture": {
|
||||
"path": "/home/zhout/project/edge_infer/models/qwen3tts_vocoder_fixture_20260924.npz",
|
||||
"bytes": 129721,
|
||||
"sha256": "dfca9e8d2a724a568a31b6cd18ff6a52fdc205fd34e13cbf1cd75f0f7937e82d"
|
||||
},
|
||||
"vocoder_ort_cpu_output": {
|
||||
"path": "/home/zhout/project/edge_infer/models/qwen3tts_vocoder_ort_cpu_output_20260924.npz",
|
||||
"bytes": 179942,
|
||||
"sha256": "8de78b94b59f01e49852f7706adbd2508a9976b8e2705cfd3b94d07bb3606cb1"
|
||||
},
|
||||
"s25_historical_gpu_output": {
|
||||
"path": "/home/zhout/project/edge_infer/vllm-omni-edge/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/s25_reference_output.npz",
|
||||
"bytes": 115235,
|
||||
"sha256": "4a26486d5e84a6e22d16f9b26f995b8fe645c29bbd70b103615ec35a0fcb10a0"
|
||||
},
|
||||
"sa8775p_tflite": {
|
||||
"path": "/home/zhout/project/edge_infer/models/qwen3tts_vocoder_sa8775p_20260924.tflite",
|
||||
"bytes": 421658132,
|
||||
"sha256": "29826da6ecc1ec2efc0e3589c3cebb42b20d7e20c0975a037d9f51625d6638cd",
|
||||
"model_id": "mnze95don",
|
||||
"compile_job_id": "jp410o3vp"
|
||||
},
|
||||
"rb3_tflite": {
|
||||
"path": "/home/zhout/project/edge_infer/models/qwen3tts_vocoder_rb3_20260924.tflite",
|
||||
"bytes": 421658132,
|
||||
"sha256": "29826da6ecc1ec2efc0e3589c3cebb42b20d7e20c0975a037d9f51625d6638cd",
|
||||
"model_id": "mn0wyzd3q",
|
||||
"compile_job_id": "j5mmy28w5"
|
||||
},
|
||||
"xelite_onnx": {
|
||||
"path": "/home/zhout/project/edge_infer/models/qwen3tts_vocoder_xelite_20260924.onnx.onnx.zip",
|
||||
"bytes": 394632184,
|
||||
"sha256": "e8f80a85d6f36248ad015edf63e553514011ae621d234a1e3d88ea71f93568f0",
|
||||
"model_id": "mnz079ezm",
|
||||
"compile_job_id": "j5qlyz8op"
|
||||
}
|
||||
},
|
||||
"fixture": {
|
||||
"shape": [
|
||||
1,
|
||||
512,
|
||||
97
|
||||
],
|
||||
"dtype": "float32",
|
||||
"min": -90.89772033691406,
|
||||
"max": 65.62191772460938,
|
||||
"mean": 0.0705118179321289,
|
||||
"std": 11.206748962402344
|
||||
},
|
||||
"cpu_reference": {
|
||||
"shape": [
|
||||
1,
|
||||
48000
|
||||
],
|
||||
"dtype": "float32",
|
||||
"rms": 0.08373439311981201
|
||||
}
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"retrieved_utc": "2026-09-23T22:21:24.623865+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "j5mmy28w5",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/j5mmy28w5/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"source_model_id": "mqyer339n",
|
||||
"target_model_id": "mn0wyzd3q",
|
||||
"device": {
|
||||
"name": "Dragonwing RB3 Gen 2 Vision Kit",
|
||||
"os": "1.6",
|
||||
"attributes": [
|
||||
"os:qc_linux",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-oe-linux-gcc11",
|
||||
"format:iot",
|
||||
"chipset:qualcomm-qcs6490",
|
||||
"chipset:qcs6490",
|
||||
"chipset:qualcomm-dragonwing-qcs6490",
|
||||
"hexagon:v68",
|
||||
"soc-model:93",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"options": "--target_runtime tflite",
|
||||
"hub_version": "aihub-2026.08.28.1"
|
||||
}
|
||||
+5361
File diff suppressed because it is too large.
Load diff
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T22:21:06.218268+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "j5mmy28w5",
|
||||
"target_model_id": "mn0wyzd3q",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"device": "Dragonwing RB3 Gen 2 Vision Kit",
|
||||
"options": "--compute_unit gpu",
|
||||
"kind": "inference",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jg9zmkv8p",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jg9zmkv8p/"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T22:21:07.854394+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "j5mmy28w5",
|
||||
"target_model_id": "mn0wyzd3q",
|
||||
"input_dataset_id": null,
|
||||
"device": "Dragonwing RB3 Gen 2 Vision Kit",
|
||||
"options": "--compute_unit gpu",
|
||||
"kind": "profile",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jp1nj707g",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jp1nj707g/"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"scope": "historical S25 TFLite GPU inference on same retained vocoder fixture; component comparison only",
|
||||
"source_model_id": "mqyer339n",
|
||||
"source_cpu_onnxruntime_version": "1.29.0",
|
||||
"s25_inference_job_id": "jpx428ojg",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"status": "component_waveform_quality_unqualified",
|
||||
"cpu_source_vs_s25_gpu": {
|
||||
"relative_l2": 0.04238531411741596,
|
||||
"snr_db": 27.455691878796923,
|
||||
"max_abs": 0.10591871291399002,
|
||||
"cosine": 0.9991021814913129,
|
||||
"reference_rms": 0.08373439205485074,
|
||||
"output_rms": 0.08376787228276501
|
||||
}
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"retrieved_utc": "2026-09-23T22:23:04.763805+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "jpx428ojg",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jpx428ojg/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"model_id": "mng7kwxon",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S25",
|
||||
"os": "15",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8-elite-for-galaxy",
|
||||
"chipset:sm8750-ac",
|
||||
"hexagon:v79",
|
||||
"soc-model:69",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"options": "--compute_unit gpu",
|
||||
"hub_version": "aihub-2026.08.28.1"
|
||||
}
|
||||
BIN
Binary file not shown.
+86
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"scope": "one Qwen3-TTS code2wav 48,000-sample component on one retained fixture; no complete TTS stream",
|
||||
"status": "component_executed_quality_unqualified",
|
||||
"device": {
|
||||
"name": "SA8775P ADP",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-android",
|
||||
"format:auto",
|
||||
"chipset:qualcomm-sa8775p",
|
||||
"hexagon:v73",
|
||||
"soc-model:52",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"compile_job_id": "jp410o3vp",
|
||||
"inference_job_id": "jgzlm8ex5",
|
||||
"profile_job_id": "j5wl78omp",
|
||||
"target_model_id": "mnze95don",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"s25_prior_inference_job_id": "jpx428ojg",
|
||||
"requested_run_options": "--compute_unit gpu",
|
||||
"files": {
|
||||
"source_onnx": {
|
||||
"bytes": 423655047,
|
||||
"sha256": "7c07666229c6d404894132e22f784525e3212a49efad55ddd532eda5286c2b2b"
|
||||
},
|
||||
"fixture": {
|
||||
"bytes": 129721,
|
||||
"sha256": "dfca9e8d2a724a568a31b6cd18ff6a52fdc205fd34e13cbf1cd75f0f7937e82d"
|
||||
},
|
||||
"cpu_output": {
|
||||
"bytes": 179942,
|
||||
"sha256": "8de78b94b59f01e49852f7706adbd2508a9976b8e2705cfd3b94d07bb3606cb1"
|
||||
},
|
||||
"s25_output": {
|
||||
"bytes": 115235,
|
||||
"sha256": "4a26486d5e84a6e22d16f9b26f995b8fe645c29bbd70b103615ec35a0fcb10a0"
|
||||
},
|
||||
"device_output": {
|
||||
"bytes": 10189,
|
||||
"sha256": "770c9c1f5a8dcaf9d8054cd2617bb8cd47d0c005c6858836452882ff3d7353d5"
|
||||
}
|
||||
},
|
||||
"cpu_source_vs_device": {
|
||||
"relative_l2": 11.832417281859046,
|
||||
"snr_db": -21.461469541638177,
|
||||
"max_abs": 1.4882186353206635,
|
||||
"cosine": 0.017841868329608154,
|
||||
"reference_rms": 0.08373439205485074,
|
||||
"output_rms": 0.988730688363501,
|
||||
"output_saturated_fraction_abs_ge_0_999": 0.9663333333333334
|
||||
},
|
||||
"s25_gpu_vs_device": {
|
||||
"relative_l2": 11.82796496222066,
|
||||
"snr_db": -21.458200587056638,
|
||||
"max_abs": 1.488037109375,
|
||||
"cosine": 0.01759117063047259,
|
||||
"reference_rms": 0.08376787228276501,
|
||||
"output_rms": 0.988730688363501,
|
||||
"output_saturated_fraction_abs_ge_0_999": 0.9663333333333334
|
||||
},
|
||||
"profile": {
|
||||
"sample_count": 100,
|
||||
"sample_min_us": 1674704,
|
||||
"nearest_rank_p50_us": 1775673,
|
||||
"nearest_rank_p95_us": 1880119,
|
||||
"reported_estimated_inference_time_us": 1674704,
|
||||
"reported_peak_memory_bytes": 361168896,
|
||||
"compute_unit_row_counts": {
|
||||
"CPU": 403,
|
||||
"GPU": 335
|
||||
}
|
||||
},
|
||||
"limits": [
|
||||
"Historical source export does not attest its exact checkpoint revision.",
|
||||
"This fixed component waveform has no listening or task-quality tolerance.",
|
||||
"Hosted component profiling excludes talker, predictor, stage handoff, streaming, admission and sustained power/thermal behavior."
|
||||
]
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"retrieved_utc": "2026-09-23T22:21:23.511773+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "jp410o3vp",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jp410o3vp/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"source_model_id": "mqyer339n",
|
||||
"target_model_id": "mnze95don",
|
||||
"device": {
|
||||
"name": "SA8775P ADP",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-android",
|
||||
"format:auto",
|
||||
"chipset:qualcomm-sa8775p",
|
||||
"hexagon:v73",
|
||||
"soc-model:52",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"options": "--target_runtime tflite",
|
||||
"hub_version": "aihub-2026.08.28.1"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T22:34:29.858166+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "jp410o3vp",
|
||||
"target_model_id": "mnze95don",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"device": "SA8775P ADP",
|
||||
"options": "--compute_unit cpu",
|
||||
"kind": "inference",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jp1nj7x7g",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jp1nj7x7g/"
|
||||
}
|
||||
BIN
Binary file not shown.
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T22:34:32.086663+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "jp410o3vp",
|
||||
"target_model_id": "mnze95don",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"device": "SA8775P ADP",
|
||||
"options": "--compute_unit gpu --tflite_options allow_fp32_as_fp16=false",
|
||||
"kind": "inference",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jgdd38lzg",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jgdd38lzg/"
|
||||
}
|
||||
+5361
File diff suppressed because it is too large.
Load diff
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T22:33:06.275073+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "jgzlm8ex5",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jgzlm8ex5/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"options": "--compute_unit gpu",
|
||||
"hub_version": "aihub-2026.09.11.0",
|
||||
"device": {
|
||||
"name": "SA8775P ADP",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-android",
|
||||
"format:auto",
|
||||
"chipset:qualcomm-sa8775p",
|
||||
"hexagon:v73",
|
||||
"soc-model:52",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"model_id": "mnze95don",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"outputs": "device_output.npz"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T22:21:01.692875+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "jp410o3vp",
|
||||
"target_model_id": "mnze95don",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"device": "SA8775P ADP",
|
||||
"options": "--compute_unit gpu",
|
||||
"kind": "inference",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jgzlm8ex5",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jgzlm8ex5/"
|
||||
}
|
||||
+5361
File diff suppressed because it is too large.
Load diff
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T22:21:03.431758+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "jp410o3vp",
|
||||
"target_model_id": "mnze95don",
|
||||
"input_dataset_id": null,
|
||||
"device": "SA8775P ADP",
|
||||
"options": "--compute_unit gpu",
|
||||
"kind": "profile",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "j5wl78omp",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/j5wl78omp/"
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"retrieved_utc": "2026-09-23T22:27:16.875471+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "j5qlyz8op",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/j5qlyz8op/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"source_model_id": "mqyer339n",
|
||||
"target_model_id": "mnz079ezm",
|
||||
"device": {
|
||||
"name": "Snapdragon X Elite CRD",
|
||||
"os": "11",
|
||||
"attributes": [
|
||||
"os:windows",
|
||||
"format:compute",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-windows",
|
||||
"chipset:qualcomm-snapdragon-x-elite",
|
||||
"chipset:sc8380xp",
|
||||
"hexagon:v73",
|
||||
"soc-model:60",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"options": "--target_runtime onnx",
|
||||
"hub_version": "aihub-2026.09.11.0"
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T22:24:46.161553+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"job_id": "j5qlyz8op",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/j5qlyz8op/",
|
||||
"source_model_id": "mqyer339n",
|
||||
"device": "Snapdragon X Elite CRD",
|
||||
"options": "--target_runtime onnx",
|
||||
"reason": "historical TFLite target rejected on Windows ARM; try ONNX runtime path"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T22:27:38.212208+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "j5qlyz8op",
|
||||
"target_model_id": "mnz079ezm",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"device": "Snapdragon X Elite CRD",
|
||||
"options": "--compute_unit cpu",
|
||||
"kind": "inference",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "j5wl78nmp",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/j5wl78nmp/"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T22:27:39.866884+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "j5qlyz8op",
|
||||
"target_model_id": "mnz079ezm",
|
||||
"input_dataset_id": null,
|
||||
"device": "Snapdragon X Elite CRD",
|
||||
"options": "--compute_unit cpu",
|
||||
"kind": "profile",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jg9zmke8p",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jg9zmke8p/"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T22:27:34.392680+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "j5qlyz8op",
|
||||
"target_model_id": "mnz079ezm",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"device": "Snapdragon X Elite CRD",
|
||||
"options": "--compute_unit gpu --onnx_execution_providers directml",
|
||||
"kind": "inference",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jpe7z83v5",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jpe7z83v5/"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T22:27:36.042380+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "j5qlyz8op",
|
||||
"target_model_id": "mnz079ezm",
|
||||
"input_dataset_id": null,
|
||||
"device": "Snapdragon X Elite CRD",
|
||||
"options": "--compute_unit gpu --onnx_execution_providers directml",
|
||||
"kind": "profile",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jgzlm8kx5",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jgzlm8kx5/"
|
||||
}
|
||||
@@ -11,6 +11,8 @@ The [WSL CPU Spark GGUF Omni profile](../../../e2e_expansion_20260923/evidence/s
|
||||
On a hosted [Galaxy S24 Spark attention-layer experiment](../../../e2e_expansion_20260923/evidence/spark_s24_full_attention/README.md), a real one-layer W8A16 QDQ artifact executed on NPU and matched prior S25 component outputs exactly on one dataset. The 100-sample layer p50/p95 was 1.109/1.205 ms; the service's 1.098 ms estimated-time field was the sample minimum. FP32-source versus S24 output hidden-state relative L2 was 1.351% without a downstream token-quality gate. This adds component C evidence only; device-local generation remains absent.
|
||||
|
||||
The same W8A16 layer also ran on hosted [Snapdragon X Elite CRD](../../../e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md) and [SA8775P ADP](../../../e2e_expansion_20260923/evidence/spark_sa8775_full_attention/README.md) NPUs, with pinned-fixture outputs identical to S24/S25; the 100-sample component p50/p95 was 1.433/1.606 ms and 1.837/2.079 ms, respectively. On [RB3 Gen 2 / QCS6490](../../../e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md), that W8A16 graph failed QNN DLC loading even after target-specific recompilation. An FP32 ONNX CPU layer then passed with source-relative L2 below 9e-7, while a calibrated W8A8 NPU layer executed but had 20.9% hidden-state relative L2 and remains numerically unqualified. These hosted component outcomes do not establish full generation or comparable device speed.
|
||||
|
||||
A [Qwen3-TTS Code2Wav follow-up](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) recovered source and target artifacts, a retained fixture, and raw historical profiles. On SA8775P, fresh GPU-requested inference saturated 96.63% of waveform samples and differed from original ONNX CPU by 11.832 relative L2; this tested vocoder path is numerically unqualified. Historical SA8775P/RB3 profile p50/p95 values were 1.751/1.874 s and 3.143/3.189 s, respectively, with both CPU and GPU rows; earlier estimated-time fields were minima. X Elite ONNX compile passed after the old TFLite rejection. Fresh profile/control/other-device jobs remain pending, so they have no new execution conclusion.
|
||||
|
||||
## Configuration × model
|
||||
|
||||
@@ -77,7 +79,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | InternVLA-A1 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Spark-X2.5 | COMPONENT_NPU_INFERENCE_PROFILED; NOT_E2E_PROFILEABLE | A real Spark-X2.5-1.7B one-layer W8A16 QDQ attention artifact, compiled as QNN DLC, ran on the exact hosted Snapdragon X Elite CRD (Windows 11 ARM64). Inference returned finite hidden/new-K/new-V tensors bitwise identical to the prior S24/S25 NPU outputs on the same fixture; the profile attributed all 73 execution-detail rows to NPU. Across 100 component samples, nearest-rank p50/p95 was 1.433/1.606 ms; the 1.346 ms estimated-time field is the minimum. Hidden/K/V relative L2 versus original FP32 ONNX CPU was 1.351%/0.377%/0.650%, without a next-token quality tolerance. There is no complete 28-layer prefill/decode/KV-ring/sampling loop, resident Omni controller or complete-request timing. | Qualify a complete X Elite resident artifact/controller with state, token quality, memory admission/cancel and sustained power; [X Elite NPU raw evidence](../../../e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md). |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | Historical QNN talker profile re-fetched successfully; no numerical/full-pipeline gate. Historical TFLite vocoder target was refused on this Windows ARM device. | Validate target-specific numerical quality, complete local execution, memory and sustained behavior. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_ONNX_VOCODER_COMPILE_PASS; NOT_E2E_PROFILEABLE | Historical QNN talker profiling exists without a numerical or full-pipeline gate. The older TFLite vocoder target was rejected on this Windows ARM device; a fresh ONNX vocoder compile passed, but its explicit DirectML GPU and CPU inference/profile jobs were pending at this snapshot. No waveform or actual placement conclusion follows from compile alone. | Retrieve and audit the pending ONNX vocoder jobs, qualify talker/predictor and complete local streaming, memory and sustained behavior; [vocoder evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md). |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
@@ -92,12 +94,12 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | Historical FP16 speech-head job re-fetched successfully; SNR 49.22 dB/top-1 match in the recorded parity test. W8A16 top-1 failed. Not complete MiniCPM-o. | Qualify thinker/encoders/vocoder and stateful local integration. |
|
||||
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Embedded SA8775P ADP / AI Hub | Spark-X2.5 | COMPONENT_NPU_INFERENCE_PROFILED; NOT_E2E_PROFILEABLE | A real Spark-X2.5-1.7B one-layer W8A16 QDQ attention artifact, compiled as QNN DLC, ran on the exact hosted SA8775P ADP (Android 14). Inference returned finite hidden/new-K/new-V tensors bitwise identical to the prior S24/S25 NPU outputs on the same fixture; the profile attributed all 73 execution-detail rows to NPU. Across 100 component samples, nearest-rank p50/p95 was 1.837/2.079 ms; the 1.749 ms estimated-time field is the minimum. Hidden/K/V relative L2 versus original FP32 ONNX CPU was 1.351%/0.377%/0.650%, without a next-token quality tolerance. There is no complete 28-layer prefill/decode/KV-ring/sampling loop, resident Omni controller or complete-request timing. | Qualify a complete embedded resident artifact/controller with state, token quality, memory admission/cancel and sustained power; [SA8775P NPU raw evidence](../../../e2e_expansion_20260923/evidence/spark_sa8775_full_attention/README.md). |
|
||||
| Embedded SA8775P ADP / AI Hub | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | Historical NPU talker and GPU vocoder profiles re-fetched successfully; no complete pipeline, parity or co-residency qualification. | Validate target-specific numerical quality, complete local execution, memory and sustained behavior. |
|
||||
| Embedded SA8775P ADP / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_VOCODER_NUMERIC_FAIL; NOT_E2E_PROFILEABLE | Historical NPU talker and mixed CPU+GPU vocoder profiles exist without a complete pipeline. A fresh GPU-requested Code2Wav inference on the retained fixture saturated 96.63% of waveform samples, with 11.832 relative L2 and -21.46 dB SNR versus original ONNX CPU; this tested vocoder path is numerically unqualified. The fresh profile has 100 samples at nearest-rank p50/p95 1.776/1.880 s with 403 CPU and 335 GPU execution-detail rows, not an all-GPU stage; it times a numerically failed output. CPU/FP32-GPU control jobs were pending. | Isolate delegate/precision failure, establish waveform and listening quality, then validate co-resident talker/predictor/vocoder, streaming, memory and sustained behavior; [raw vocoder evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md). |
|
||||
| Embedded SA8775P ADP / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Embedded SA8775P ADP / AI Hub | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Embedded SA8775P ADP / AI Hub | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | Spark-X2.5 | COMPONENT_CPU_PARITY_PASS; COMPONENT_NPU_NUMERIC_UNQUALIFIED; NOT_E2E_PROFILEABLE | On the exact hosted Dragonwing RB3 Gen 2 Vision Kit (QCS6490, QC Linux 1.6), the reused S24 W8A16 QNN DLC and a separately RB3-compiled W8A16 DLC each failed inference/profile at QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR. An FP32 ONNX one-layer CPU route passed with all 56 profile rows on CPU, source-relative hidden/K/V L2 of 8.73e-7/2.98e-7/3.37e-7 and 100-sample component p50/p95 19.304/21.113 ms. A calibrated W8A8 QNN DLC ran with all 61 rows on NPU and p50/p95 5.610/5.761 ms, but hidden/K/V relative L2 was 20.914%/1.702%/4.413% versus FP32 CPU, without a next-token quality gate. These are different precisions, not a qualified speedup; no complete 28-layer prefill/decode/KV-ring/sampling, local Omni controller or complete-request timing exists. | Improve representative W8A8 calibration or find another compatible quality-passing artifact; validate token parity, then full device-resident generation, admission/cancel and sustained power; [RB3 raw evidence](../../../e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md). |
|
||||
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | Historical GPU vocoder profile re-fetched successfully. The attempted NPU talker build was rejected for floating-point inputs. No full model pipeline. | Validate target-specific numerical quality, complete local execution, memory and sustained behavior. |
|
||||
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_VOCODER_PROFILE_ONLY; NOT_E2E_PROFILEABLE | The historical TFLite vocoder profile re-fetched with 100 samples at nearest-rank p50/p95 3.143/3.189 s and 403 CPU plus 335 GPU execution-detail rows. Its estimated-time field 3.107 s is the minimum, not median. Fresh same-fixture inference/profile was pending, so numerical quality is unknown. The historical NPU talker build was rejected for floating-point inputs; no full model pipeline exists. | Retrieve fresh vocoder output and placement, establish a quality-passing integer talker artifact, then validate complete local execution, memory and sustained behavior; [vocoder evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md). |
|
||||
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
|
||||
Reference in new issue
Block a user