diff --git a/benchmarks/edge_harness/experiments/audit_minicpmo_tts_qualcomm.py b/benchmarks/edge_harness/experiments/audit_minicpmo_tts_qualcomm.py index aa0c5ff4e..f7d3fcb44 100644 --- a/benchmarks/edge_harness/experiments/audit_minicpmo_tts_qualcomm.py +++ b/benchmarks/edge_harness/experiments/audit_minicpmo_tts_qualcomm.py @@ -17,6 +17,7 @@ REFERENCE_SHA256 = "902adcceadbe761d938ef2ade51cf953c8917ea1abfaee33a10c84e300c1 COMPILE_OPTIONS = { "--target_runtime qnn_context_binary --quantize_full_type float16", "--target_runtime qnn_dlc --qnn_options default_graph_htp_precision=FLOAT16", + "--target_runtime tflite", } INPUT_NAMES = ["x", "cos", "sin", "mask"] + [ name for layer in range(20) for name in (f"k_cache_{layer}", f"v_cache_{layer}") @@ -73,6 +74,15 @@ def main() -> None: raise ValueError("compile source/options differ from the pinned speech-head artifact") if inferred.get("model_id") != compiled.get("target_model_id"): raise ValueError("inference used a different target artifact") + if compiled["options"] == "--target_runtime tflite": + allowed_run_options = {"--compute_unit cpu"} + elif compiled["options"].startswith("--target_runtime qnn_dlc"): + allowed_run_options = {"--compute_unit npu"} + else: + # Historical S24 context-binary inference did not record a run option. + allowed_run_options = {"", "--compute_unit npu"} + if inferred.get("options") not in allowed_run_options: + raise ValueError("inference compute request differs from the compiled target route") if inferred.get("input_dataset_id") not in {"d7gwxe3y2", "d7zn8rk57"}: raise ValueError("inference did not use a retained matching fixture dataset") if (compiled.get("device", {}).get("name") != args.device_name @@ -143,7 +153,7 @@ def main() -> None: "limits": [ "The historical ONNX export does not attest the exact source checkpoint revision.", "One retained decode fixture cannot establish speech-token or generated-audio quality.", - "Actual NPU placement requires a device profile; a request or QNN artifact alone is insufficient.", + "Actual CPU/NPU placement requires a device profile; a request or compiled artifact alone is insufficient.", "Thinker, audio/vision encoders, vocoder, state loop and complete Omni request were not tested.", ], } diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260924/README.md index 610995d1a..47505ec0c 100644 --- a/benchmarks/edge_harness/results/e2e_expansion_20260924/README.md +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/README.md @@ -4,8 +4,8 @@ The [Qwen3-TTS vocoder study](evidence/qwen_tts_vocoder_qualcomm/README.md) reus A local HX370 x86-64 LiteRT CPU control of the identical SA/RB3 TFLite binary matched the original ONNX CPU waveform within 3.45e-6 relative L2. The SA saturation is therefore not universal to the converted artifact; the exact SA CPU control passed numerical and placement checks, while the faulty GPU delegate operation remains unidentified. -The [talker decode-step attempt](evidence/qwen_tts_talker_qualcomm/README.md) pins a separate 28-layer source and synthetic cache fixture. Historical FP16 QNN context-binary inference executed on S24, S25, SA8775P and X Elite, but all four changed the CPU top-1 speech token with 8.57–9.43% logits relative L2. These artifacts are numerically unqualified on the fixture. RB3 rejected the floating-point-input artifact. Explicit X Elite ONNX CPU and SA8775P TFLite CPU fallback compiles passed, with same-fixture inference pending. No complete talker loop or TTS stream passed. +The [talker decode-step attempt](evidence/qwen_tts_talker_qualcomm/README.md) pins a separate 28-layer source and synthetic cache fixture. Historical FP16 QNN context-binary inference executed on S24, S25, SA8775P and X Elite, but all four changed the CPU top-1 speech token with 8.57–9.43% logits relative L2. These artifacts are numerically unqualified on the fixture. RB3 rejected the floating-point-input artifact. Explicit X Elite ONNX CPU and SA8775P TFLite CPU fallback compiles passed. X Elite CPU-requested inference preserved CPU top-1 token 80 with logits relative L2 2.07e-5; its placement profile and SA inference remain pending. No complete talker loop or TTS stream passed. -The [MiniCPM-o speech-head study](evidence/minicpmo_tts_qualcomm/README.md) recovered the real 20-layer ONNX source and a matching retained fixture. S24 and S25 FP16 context-binary outputs preserved the ONNX CPU top-1 speech token with 0.346%/0.360% logits relative L2 on one step. FP16 QNN DLC compiles passed for SA8775P, X Elite and RB3. SA8775P NPU-requested inference preserved CPU top-1 token 1867 with 0.3568% logits relative L2; X Elite same-fixture inference also preserved token 1867 with 0.3170% logits relative L2. SA and X Elite placement profiles remain pending. RB3 FP16 QNN inference failed at graph load despite compile success; an explicit TFLite CPU fallback compile is pending. The complete multimodal model remains unqualified on those devices. +The [MiniCPM-o speech-head study](evidence/minicpmo_tts_qualcomm/README.md) recovered the real 20-layer ONNX source and a matching retained fixture. S24 and S25 FP16 context-binary outputs preserved the ONNX CPU top-1 speech token with 0.346%/0.360% logits relative L2 on one step. FP16 QNN DLC compiles passed for SA8775P, X Elite and RB3. SA8775P NPU-requested inference preserved CPU top-1 token 1867 with 0.3568% logits relative L2; X Elite same-fixture inference also preserved token 1867 with 0.3170% logits relative L2. SA and X Elite placement profiles remain pending. RB3 FP16 QNN inference failed at graph load despite compile success; an explicit TFLite CPU fallback compile passed with same-fixture CPU-requested inference pending. The complete multimodal model remains unqualified on those devices. The [InternVLA Cosmos S25 attempt](evidence/internvla_cosmos_qualcomm/README.md) repackages the real policy's image encoder from its parity-tested ONNX export. FLOAT16 QNN DLC inference returned a latent within 0.0524% relative L2 of the source on one pattern. Injecting it into the real policy on local CPU changed synthetic actions by 0.742% relative L2, without a task tolerance. S25 NPU placement profiling was pending. The same encoder compiled to FP16 QNN DLC for X Elite CRD, with same-fixture NPU-requested inference pending. Device-local action-policy support remains unqualified. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md index cd974c11c..ca0d42ae4 100644 --- a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md @@ -6,6 +6,6 @@ The [S25 historical inference](s25/inference_report.json) used dataset `d7gwxe3y The historical [S24](s24/compile_report.json) and [S25](s25/compile_report.json) FP16 QNN context-binary compiles and inference jobs succeeded. Against the **same original ONNX CPU fixture**, [S24](s24/audit_report.json) logits relative L2 was **0.3459%** and max K/V relative L2 **1.235%**; [S25](s25/audit_report.json) logits relative L2 was **0.3605%** and max K/V relative L2 **1.311%**. Both preserved the CPU top-1 speech token **1867** on this one synthetic step. The raw [S24](s24/device_output.npz) and [S25](s25/device_output.npz) arrays are retained. [S24](s24/profile_submission.json) and [S25](s25/profile_submission.json) placement profiles were submitted and pending. These values remain numerical component evidence, without a generated-audio quality threshold or actual NPU placement proof. -New FP16 QNN DLC [SA8775P](sa8775p/compile_report.json), [Snapdragon X Elite](xelite/compile_report.json) and [RB3 Gen 2](rb3/compile_report.json) compiles passed from the same source. The SA8775P [NPU-requested inference](sa8775p/inference_report.json) returned all 41 finite tensors on the retained fixture. Its [audit](sa8775p/audit_report.json) found **0.3568% logits relative L2** versus ONNX CPU, preserved CPU top-1 speech token **1867**, and measured maximum K/V relative L2 **1.225%**. The [placement profile](sa8775p/profile_submission.json) was submitted and remains pending; the inference request alone does not prove actual NPU node assignment. The X Elite [NPU-requested inference](xelite/inference_report.json) also returned all 41 finite tensors: its [audit](xelite/audit_report.json) found **0.3170% logits relative L2**, preserved CPU top-1 token **1867**, and measured maximum K/V relative L2 **1.221%**. Its [placement profile](xelite/profile_submission.json) was submitted and remains pending. The exact RB3 [FP16 QNN DLC inference](rb3/inference_report.json) **failed** while loading the graph with QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR, despite a successful compile. RB3 does not advertise FP16 HTP support in the device attributes; this failure constrains the tested artifact, not every RB3 path. An explicit [TFLite CPU fallback compile](rb3/cpu_fallback/compile_submission.json) was submitted and remains pending. The [collector](../../../../experiments/collect_aihub_component.py) retrieves terminal jobs, and the [auditor](../../../../experiments/audit_minicpmo_tts_qualcomm.py) pins hashes and checks all 41 output tensors. +New FP16 QNN DLC [SA8775P](sa8775p/compile_report.json), [Snapdragon X Elite](xelite/compile_report.json) and [RB3 Gen 2](rb3/compile_report.json) compiles passed from the same source. The SA8775P [NPU-requested inference](sa8775p/inference_report.json) returned all 41 finite tensors on the retained fixture. Its [audit](sa8775p/audit_report.json) found **0.3568% logits relative L2** versus ONNX CPU, preserved CPU top-1 speech token **1867**, and measured maximum K/V relative L2 **1.225%**. The [placement profile](sa8775p/profile_submission.json) was submitted and remains pending; the inference request alone does not prove actual NPU node assignment. The X Elite [NPU-requested inference](xelite/inference_report.json) also returned all 41 finite tensors: its [audit](xelite/audit_report.json) found **0.3170% logits relative L2**, preserved CPU top-1 token **1867**, and measured maximum K/V relative L2 **1.221%**. Its [placement profile](xelite/profile_submission.json) was submitted and remains pending. The exact RB3 [FP16 QNN DLC inference](rb3/inference_report.json) **failed** while loading the graph with QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR, despite a successful compile. RB3 does not advertise FP16 HTP support in the device attributes; this failure constrains the tested artifact, not every RB3 path. An explicit [TFLite CPU fallback compile](rb3/cpu_fallback/compile_report.json) succeeded, producing target mnwv5d7wq; [same-fixture CPU-requested inference](rb3/cpu_fallback/inference_submission.json) is pending. The [collector](../../../../experiments/collect_aihub_component.py) retrieves terminal jobs, and the [auditor](../../../../experiments/audit_minicpmo_tts_qualcomm.py) pins hashes and checks all 41 output tensors. Next: audit those device outputs and placement, then qualify the entire model with real multimodal inputs, stateful speech generation, task/audio quality, memory admission, cancellation and sustained power/thermal behavior. AI Hub is the test facility, not a deployment dependency. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/rb3/cpu_fallback/compile_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/rb3/cpu_fallback/compile_report.json new file mode 100644 index 000000000..cca47e99b --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/rb3/cpu_fallback/compile_report.json @@ -0,0 +1,31 @@ +{ + "retrieved_utc": "2026-09-24T00:23:45.508631+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "job_id": "j5m0m8oqg", + "url": "https://workbench.aihub.qualcomm.com/jobs/j5m0m8oqg/", + "status": "SUCCESS", + "status_message": "", + "source_model_id": "mn4gjjpvn", + "target_model_id": "mnwv5d7wq", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "options": "--target_runtime tflite", + "hub_version": "aihub-2026.09.11.0" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/rb3/cpu_fallback/inference_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/rb3/cpu_fallback/inference_submission.json new file mode 100644 index 000000000..524855ca9 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/rb3/cpu_fallback/inference_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-24T00:23:47.267012+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "j5m0m8oqg", + "target_model_id": "mnwv5d7wq", + "input_dataset_id": "d7gwxe3y2", + "device": "Dragonwing RB3 Gen 2 Vision Kit", + "options": "--compute_unit cpu", + "kind": "inference", + "status": "SUBMITTED", + "job_id": "jp4y13yvp", + "url": "https://workbench.aihub.qualcomm.com/jobs/jp4y13yvp/" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md index b72540f69..ebe0b5796 100644 --- a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md @@ -6,7 +6,7 @@ The deterministic synthetic input dataset `d7dw6pgy2` has SHA256 `0bf538e1b64dfc Historical FP16 QNN context-binary [S24](s24/compile_report.json), [S25](s25/compile_report.json), [SA8775P](sa8775p/compile_report.json), and [Snapdragon X Elite](xelite/compile_report.json) compile jobs succeeded. Same-fixture NPU-requested inference succeeded on all four, but **all changed the CPU reference top-1 logit** on this synthetic step: CPU token 80 versus S24/SA/X Elite 1720 and S25 412. The [S24](s24/audit_report.json), [S25](s25/audit_report.json), [SA8775P](sa8775p/audit_report.json) and [X Elite](xelite/audit_report.json) audits report logits relative L2 **9.33%/8.99%/9.43%/8.57%**, respectively, with maximum K/V relative L2 **11.23%/11.71%/11.54%/11.50%**. These FP16 artifacts are numerically unqualified for this fixture; a successful graph execution is not speech-token parity. The [RB3 compile](rb3/compile_report.json) rejected floating-point input on this device. These are artifact-specific outcomes, not full-model support claims. -Explicit [X Elite ONNX CPU](xelite/cpu_fallback/compile_report.json) and [SA8775P TFLite CPU](sa8775p/cpu_fallback/compile_report.json) fallback compiles succeeded from the same source. Same-fixture [X Elite](xelite/cpu_fallback/inference_submission.json) and [SA8775P](sa8775p/cpu_fallback/inference_submission.json) CPU-requested inference jobs were pending at this snapshot. Neither compile alone is a numerical pass or a full TTS deployment. +Explicit [X Elite ONNX CPU](xelite/cpu_fallback/compile_report.json) and [SA8775P TFLite CPU](sa8775p/cpu_fallback/compile_report.json) fallback compiles succeeded from the same source. The same-fixture [X Elite CPU-requested inference](xelite/cpu_fallback/inference_report.json) succeeded. Its [audit](xelite/cpu_fallback/audit_report.json) found logits relative L2 2.07e-5, maximum new-K/V relative L2 2.18e-5, and preserved ONNX CPU top-1 speech token 80. A [CPU-requested placement profile](xelite/cpu_fallback/profile_submission.json) was submitted and remains pending. [SA8775P CPU-requested inference](sa8775p/cpu_fallback/inference_submission.json) also remains pending. This is one numerically close decode step, not a talker loop or full TTS deployment. The [collector](../../../../experiments/collect_aihub_component.py) retrieves terminal inference outputs without resubmission. The [tensor auditor](../../../../experiments/audit_qwen_tts_talker_qualcomm.py) enforces the source, fixture, compile target, exact device, output shape and finiteness contracts, and reports logits top-1 plus relative error for all 56 new K/V tensors. It does not convert those metrics to a quality pass without an explicit token/audio tolerance. Its 57-output contract was exercised against the pinned CPU reference as a local self-check; that self-check is not device evidence. Requested NPU placement still needs a profile. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/xelite/cpu_fallback/audit_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/xelite/cpu_fallback/audit_report.json new file mode 100644 index 000000000..9ccb4d74f --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/xelite/cpu_fallback/audit_report.json @@ -0,0 +1,400 @@ +{ + "scope": "one synthetic fixed-cache 28-layer Qwen3-TTS talker decode step; no complete TTS stream", + "status": "component_executed_numeric_measured", + "device": { + "name": "Snapdragon X Elite CRD", + "os": "11", + "attributes": [ + "os:windows", + "format:compute", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-windows", + "chipset:qualcomm-snapdragon-x-elite", + "chipset:sc8380xp", + "hexagon:v73", + "soc-model:60", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "compile_job_id": "jp1njv18g", + "inference_job_id": "jgzlm40o5", + "source_model_id": "mq3xre4rq", + "target_model_id": "mnwv5rp3q", + "input_dataset_id": "d7dw6pgy2", + "requested_run_options": "--compute_unit cpu", + "files": { + "source_onnx": { + "bytes": 1776377476, + "sha256": "41feb08517678e94d9301e992d0df610fc60e7c9faba5600703298c86554fd22" + }, + "fixture": { + "bytes": 54591500, + "sha256": "0bf538e1b64dfc9737dc4f925d61c6e4ffa5075a5dcd7afb1e85ac3bd2d9afea" + }, + "cpu_reference": { + "bytes": 237615, + "sha256": "ee0ead70e33c02c7d3d6bcb757c5cf0fcbb9eb6fc75c34abfbe8a4479657d5be" + }, + "device_output": { + "bytes": 238145, + "sha256": "12c90bfe694f88163db4126494bf99fc93b0c086ff6ce4b2e454f00c1b49b284" + } + }, + "tensor_comparison": { + "logits": { + "relative_l2": 2.070890921519086e-05, + "max_abs_error": 0.0003037452697753906, + "reference_rms": 3.7322664515389006, + "output_rms": 3.732294970760747, + "cpu_top1": 80, + "device_top1": 80, + "top1_matches": true + }, + "k_new_0": { + "relative_l2": 3.3991528815743865e-07, + "max_abs_error": 1.6689300537109375e-06, + "reference_rms": 1.1349352034374294, + "output_rms": 1.1349352771003447 + }, + "v_new_0": { + "relative_l2": 5.698665801439263e-07, + "max_abs_error": 3.2782554626464844e-07, + "reference_rms": 0.1383137764927112, + "output_rms": 0.1383138417993723 + }, + "k_new_1": { + "relative_l2": 3.568658071574065e-07, + "max_abs_error": 2.384185791015625e-06, + "reference_rms": 1.4773734279001252, + "output_rms": 1.4773733425698152 + }, + "v_new_1": { + "relative_l2": 3.6938187848675376e-07, + "max_abs_error": 2.5331974029541016e-07, + "reference_rms": 0.162893856079411, + "output_rms": 0.16289384385106143 + }, + "k_new_2": { + "relative_l2": 6.35958056844424e-07, + "max_abs_error": 4.0531158447265625e-06, + "reference_rms": 1.4749251890370425, + "output_rms": 1.4749251951999462 + }, + "v_new_2": { + "relative_l2": 6.793962536784832e-07, + "max_abs_error": 7.152557373046875e-07, + "reference_rms": 0.2996575863196337, + "output_rms": 0.2996575800817967 + }, + "k_new_3": { + "relative_l2": 9.409087965415886e-07, + "max_abs_error": 4.76837158203125e-06, + "reference_rms": 1.4529848472712377, + "output_rms": 1.452984775887525 + }, + "v_new_3": { + "relative_l2": 9.505263309197924e-07, + "max_abs_error": 9.5367431640625e-07, + "reference_rms": 0.2917197159228712, + "output_rms": 0.2917197093499009 + }, + "k_new_4": { + "relative_l2": 1.0706016650770476e-06, + "max_abs_error": 8.64267349243164e-06, + "reference_rms": 1.5413980317857385, + "output_rms": 1.5413980508633573 + }, + "v_new_4": { + "relative_l2": 1.188899898476672e-06, + "max_abs_error": 2.2649765014648438e-06, + "reference_rms": 0.41222657123793377, + "output_rms": 0.41222662458456166 + }, + "k_new_5": { + "relative_l2": 1.2643051348915694e-06, + "max_abs_error": 1.049041748046875e-05, + "reference_rms": 1.6285454176549035, + "output_rms": 1.6285454822857448 + }, + "v_new_5": { + "relative_l2": 1.5995047083181534e-06, + "max_abs_error": 1.9073486328125e-06, + "reference_rms": 0.32638470026742417, + "output_rms": 0.3263845448937799 + }, + "k_new_6": { + "relative_l2": 1.5023569689595248e-06, + "max_abs_error": 1.33514404296875e-05, + "reference_rms": 1.8000231156167423, + "output_rms": 1.8000231830423972 + }, + "v_new_6": { + "relative_l2": 1.8566273706768326e-06, + "max_abs_error": 1.6838312149047852e-06, + "reference_rms": 0.2629708328253682, + "output_rms": 0.2629708646262636 + }, + "k_new_7": { + "relative_l2": 1.743420510352378e-06, + "max_abs_error": 1.3828277587890625e-05, + "reference_rms": 1.6255722851184644, + "output_rms": 1.625572354626315 + }, + "v_new_7": { + "relative_l2": 1.8796305186543377e-06, + "max_abs_error": 2.8908252716064453e-06, + "reference_rms": 0.348368074126628, + "output_rms": 0.3483682630576819 + }, + "k_new_8": { + "relative_l2": 2.035641431943173e-06, + "max_abs_error": 1.2874603271484375e-05, + "reference_rms": 1.516512353065435, + "output_rms": 1.5165123695421003 + }, + "v_new_8": { + "relative_l2": 2.2009327137930583e-06, + "max_abs_error": 2.391636371612549e-06, + "reference_rms": 0.28027916002007797, + "output_rms": 0.2802791698498317 + }, + "k_new_9": { + "relative_l2": 2.0543037439114592e-06, + "max_abs_error": 1.3142824172973633e-05, + "reference_rms": 1.5960043895576705, + "output_rms": 1.5960044635114818 + }, + "v_new_9": { + "relative_l2": 2.7421143373469114e-06, + "max_abs_error": 3.5762786865234375e-06, + "reference_rms": 0.37541008682864946, + "output_rms": 0.37540997627074507 + }, + "k_new_10": { + "relative_l2": 2.388894208897564e-06, + "max_abs_error": 1.430511474609375e-05, + "reference_rms": 1.6561161468173449, + "output_rms": 1.6561160819303562 + }, + "v_new_10": { + "relative_l2": 3.1622230448221883e-06, + "max_abs_error": 3.285706043243408e-06, + "reference_rms": 0.26429778504066537, + "output_rms": 0.2642977737488034 + }, + "k_new_11": { + "relative_l2": 2.8406429780760036e-06, + "max_abs_error": 1.7017126083374023e-05, + "reference_rms": 1.5170462775366995, + "output_rms": 1.517046260157504 + }, + "v_new_11": { + "relative_l2": 3.5868736729562148e-06, + "max_abs_error": 2.7194619178771973e-06, + "reference_rms": 0.22548908846990579, + "output_rms": 0.22548903412301166 + }, + "k_new_12": { + "relative_l2": 3.0764719761741945e-06, + "max_abs_error": 1.913309097290039e-05, + "reference_rms": 1.5731131793440647, + "output_rms": 1.5731130872979318 + }, + "v_new_12": { + "relative_l2": 4.0830478000934314e-06, + "max_abs_error": 4.06801700592041e-06, + "reference_rms": 0.20839677682266325, + "output_rms": 0.20839677591079137 + }, + "k_new_13": { + "relative_l2": 3.818613672751543e-06, + "max_abs_error": 2.2172927856445312e-05, + "reference_rms": 1.4939452330557514, + "output_rms": 1.4939452577856214 + }, + "v_new_13": { + "relative_l2": 4.77994361885633e-06, + "max_abs_error": 4.276633262634277e-06, + "reference_rms": 0.24929462665621008, + "output_rms": 0.24929473233780286 + }, + "k_new_14": { + "relative_l2": 3.3851026163113835e-06, + "max_abs_error": 2.0503997802734375e-05, + "reference_rms": 1.4942429579997731, + "output_rms": 1.494242929117275 + }, + "v_new_14": { + "relative_l2": 5.273689682240653e-06, + "max_abs_error": 5.7443976402282715e-06, + "reference_rms": 0.32824019866053383, + "output_rms": 0.32824043167321204 + }, + "k_new_15": { + "relative_l2": 4.6078336232156544e-06, + "max_abs_error": 3.0368566513061523e-05, + "reference_rms": 1.6561072116211841, + "output_rms": 1.6561074926088477 + }, + "v_new_15": { + "relative_l2": 6.551747528405165e-06, + "max_abs_error": 5.8710575103759766e-06, + "reference_rms": 0.24976283574047556, + "output_rms": 0.24976354565102096 + }, + "k_new_16": { + "relative_l2": 5.7220585365545366e-06, + "max_abs_error": 4.029273986816406e-05, + "reference_rms": 1.617983543941328, + "output_rms": 1.6179831449374849 + }, + "v_new_16": { + "relative_l2": 7.365807812863625e-06, + "max_abs_error": 9.655952453613281e-06, + "reference_rms": 0.3826095020438836, + "output_rms": 0.38261009659668105 + }, + "k_new_17": { + "relative_l2": 8.792203745745486e-06, + "max_abs_error": 6.508827209472656e-05, + "reference_rms": 1.6828817809060599, + "output_rms": 1.6828821469624746 + }, + "v_new_17": { + "relative_l2": 9.271391035622246e-06, + "max_abs_error": 1.570582389831543e-05, + "reference_rms": 0.4656876133547257, + "output_rms": 0.4656879339538229 + }, + "k_new_18": { + "relative_l2": 9.700928334954714e-06, + "max_abs_error": 6.389617919921875e-05, + "reference_rms": 1.5546463592216835, + "output_rms": 1.5546463372457173 + }, + "v_new_18": { + "relative_l2": 1.1158471683354893e-05, + "max_abs_error": 1.773238182067871e-05, + "reference_rms": 0.5200020604619255, + "output_rms": 0.5200023134045119 + }, + "k_new_19": { + "relative_l2": 1.0682088176024872e-05, + "max_abs_error": 0.00010627880692481995, + "reference_rms": 1.5217070844504548, + "output_rms": 1.5217069734030593 + }, + "v_new_19": { + "relative_l2": 1.2651436529223935e-05, + "max_abs_error": 2.181529998779297e-05, + "reference_rms": 0.5402920798790236, + "output_rms": 0.5402926222555624 + }, + "k_new_20": { + "relative_l2": 1.1064903705528427e-05, + "max_abs_error": 0.00011444091796875, + "reference_rms": 1.6813854859968058, + "output_rms": 1.6813857330107715 + }, + "v_new_20": { + "relative_l2": 1.4200971069100998e-05, + "max_abs_error": 2.631545066833496e-05, + "reference_rms": 0.6088730895277438, + "output_rms": 0.6088725799616966 + }, + "k_new_21": { + "relative_l2": 1.2390564268305158e-05, + "max_abs_error": 0.0001004934310913086, + "reference_rms": 1.7370493419005983, + "output_rms": 1.7370489880147637 + }, + "v_new_21": { + "relative_l2": 1.5827906442139433e-05, + "max_abs_error": 4.9114227294921875e-05, + "reference_rms": 0.6072011016139925, + "output_rms": 0.6072015267581002 + }, + "k_new_22": { + "relative_l2": 1.3547088934561051e-05, + "max_abs_error": 0.0001615285873413086, + "reference_rms": 1.9661056736696203, + "output_rms": 1.9661056197755753 + }, + "v_new_22": { + "relative_l2": 1.6808666574657083e-05, + "max_abs_error": 4.470348358154297e-05, + "reference_rms": 0.7362191931025958, + "output_rms": 0.7362195471846572 + }, + "k_new_23": { + "relative_l2": 1.5109735898958623e-05, + "max_abs_error": 8.94889235496521e-05, + "reference_rms": 1.6005997999291703, + "output_rms": 1.6005997456598573 + }, + "v_new_23": { + "relative_l2": 1.9473568443103025e-05, + "max_abs_error": 5.733966827392578e-05, + "reference_rms": 0.6675198669572269, + "output_rms": 0.6675188832283546 + }, + "k_new_24": { + "relative_l2": 1.5749334480637003e-05, + "max_abs_error": 9.369850158691406e-05, + "reference_rms": 1.6337684549178757, + "output_rms": 1.6337690246224614 + }, + "v_new_24": { + "relative_l2": 2.091083594678951e-05, + "max_abs_error": 5.21540641784668e-05, + "reference_rms": 0.6240472225015637, + "output_rms": 0.6240464838495503 + }, + "k_new_25": { + "relative_l2": 1.3746349352124886e-05, + "max_abs_error": 9.036064147949219e-05, + "reference_rms": 1.527601662385539, + "output_rms": 1.5276026196401622 + }, + "v_new_25": { + "relative_l2": 1.89076446258063e-05, + "max_abs_error": 6.270408630371094e-05, + "reference_rms": 0.9133071626950193, + "output_rms": 0.9133069102315201 + }, + "k_new_26": { + "relative_l2": 1.5391140002971875e-05, + "max_abs_error": 8.511543273925781e-05, + "reference_rms": 1.4905787414732559, + "output_rms": 1.4905785746246762 + }, + "v_new_26": { + "relative_l2": 2.1793747793817946e-05, + "max_abs_error": 5.364418029785156e-05, + "reference_rms": 0.7009595279765859, + "output_rms": 0.7009595271174758 + }, + "k_new_27": { + "relative_l2": 1.6046207815528483e-05, + "max_abs_error": 0.00010585784912109375, + "reference_rms": 1.5453936433107067, + "output_rms": 1.5453934578541693 + }, + "v_new_27": { + "relative_l2": 2.0308666421389636e-05, + "max_abs_error": 7.164478302001953e-05, + "reference_rms": 0.6731545154533489, + "output_rms": 0.6731552039308182 + } + }, + "cache_max_relative_l2": 2.1793747793817946e-05, + "limits": [ + "The historical source export does not attest an exact checkpoint revision.", + "The fixture is synthetic and has no token-to-audio quality tolerance.", + "Requested compute unit is not actual placement without a device profile.", + "No predictor, vocoder, persistent cache loop or complete TTS stream was tested." + ] +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/xelite/cpu_fallback/device_output.npz b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/xelite/cpu_fallback/device_output.npz new file mode 100644 index 000000000..d491b1c93 Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/xelite/cpu_fallback/device_output.npz differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/xelite/cpu_fallback/inference_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/xelite/cpu_fallback/inference_report.json new file mode 100644 index 000000000..fd0b7b0ed --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/xelite/cpu_fallback/inference_report.json @@ -0,0 +1,32 @@ +{ + "timestamp_utc": "2026-09-24T00:25:11.672965+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jgzlm40o5", + "url": "https://workbench.aihub.qualcomm.com/jobs/jgzlm40o5/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit cpu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Snapdragon X Elite CRD", + "os": "11", + "attributes": [ + "os:windows", + "format:compute", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-windows", + "chipset:qualcomm-snapdragon-x-elite", + "chipset:sc8380xp", + "hexagon:v73", + "soc-model:60", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "model_id": "mnwv5rp3q", + "input_dataset_id": "d7dw6pgy2", + "outputs": "device_output.npz" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/xelite/cpu_fallback/profile_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/xelite/cpu_fallback/profile_submission.json new file mode 100644 index 000000000..575083163 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/xelite/cpu_fallback/profile_submission.json @@ -0,0 +1,13 @@ +{ + "timestamp_utc": "2026-09-24T00:25:46.492184+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jp1njv18g", + "target_model_id": "mnwv5rp3q", + "device": "Snapdragon X Elite CRD", + "options": "--compute_unit cpu", + "kind": "profile", + "status": "SUBMITTED", + "job_id": "jp2rwed4g", + "url": "https://workbench.aihub.qualcomm.com/jobs/jp2rwed4g/" +} diff --git a/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md b/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md index 3ba6a7e4b..7ec7169ae 100644 --- a/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md +++ b/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md @@ -16,11 +16,11 @@ A [Qwen3-TTS Code2Wav follow-up](../../../e2e_expansion_20260924/evidence/qwen_t The identical SA/RB3 TFLite binary matched original ONNX CPU within 3.45e-6 relative L2 in a separate local HX370 x86-64 LiteRT CPU control. This narrows the SA waveform saturation to hosted device/runtime behavior, without proving either embedded CPU route or a specific GPU-delegate fault. -A separate [Qwen3-TTS talker decode-step attempt](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) recovered a 28-layer ONNX export and synthetic fixed-cache fixture. Historical FP16 QNN context-binary inference executed on S24, S25, SA8775P and X Elite, but all four changed the CPU reference top-1 speech token with 8.57–9.43% logits relative L2; they are numerically unqualified on this fixture. An RB3 compile rejected floating-point input. No complete talker loop or TTS stream follows. +A separate [Qwen3-TTS talker decode-step attempt](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) recovered a 28-layer ONNX export and synthetic fixed-cache fixture. Historical FP16 QNN context-binary inference executed on S24, S25, SA8775P and X Elite, but all four changed the CPU reference top-1 speech token with 8.57–9.43% logits relative L2; they are numerically unqualified on this fixture. An RB3 compile rejected floating-point input. A separate X Elite ONNX CPU-requested talker step preserved CPU token 80 with logits relative L2 2.07e-5; placement profiling is pending. No complete talker loop or TTS stream follows. A [Galaxy S25 InternVLA Cosmos attempt](../../../e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md) packaged the real Place_Markpen image encoder into one ONNX file after two CPU parity fixtures. FLOAT16 QNN DLC inference returned a latent within 0.0524% relative L2 of the source on one pattern; device placement profile was pending. Injecting that latent into the real policy on local CPU changed synthetic actions by 0.742% relative L2, without a task tolerance. The same encoder compiled to FP16 QNN DLC for X Elite CRD, with same-fixture NPU-requested inference pending. This advances component and action-sensitivity evidence, not device-local full-policy qualification. -A [MiniCPM-o speech-head study](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md) recovered the historical 20-layer source and matching S24/S25 fixtures. Both FP16 context-binary outputs preserved the ONNX CPU top-1 speech token on this one decode step with logits relative L2 of 0.346%/0.360%. New FP16 QNN DLC compiles passed for SA8775P, X Elite and RB3. SA8775P same-fixture NPU-requested inference returned finite outputs, preserved CPU top-1 token 1867 and had 0.3568% logits relative L2; actual NPU placement profiling is pending. X Elite same-fixture inference also preserved token 1867 with 0.3170% logits relative L2; its placement profile remains pending; RB3 FP16 QNN inference failed at graph load, and an explicit CPU fallback compile is pending. These are speech-head component results, not multimodal MiniCPM-o requests. +A [MiniCPM-o speech-head study](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md) recovered the historical 20-layer source and matching S24/S25 fixtures. Both FP16 context-binary outputs preserved the ONNX CPU top-1 speech token on this one decode step with logits relative L2 of 0.346%/0.360%. New FP16 QNN DLC compiles passed for SA8775P, X Elite and RB3. SA8775P same-fixture NPU-requested inference returned finite outputs, preserved CPU top-1 token 1867 and had 0.3568% logits relative L2; actual NPU placement profiling is pending. X Elite same-fixture inference also preserved token 1867 with 0.3170% logits relative L2; its placement profile remains pending; RB3 FP16 QNN inference failed at graph load, and an explicit TFLite CPU fallback compile passed with same-fixture inference pending. These are speech-head component results, not multimodal MiniCPM-o requests. ## Configuration × model @@ -87,7 +87,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i | PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | InternVLA-A1 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | PC Snapdragon X Elite CRD / AI Hub | Spark-X2.5 | COMPONENT_NPU_INFERENCE_PROFILED; NOT_E2E_PROFILEABLE | A real Spark-X2.5-1.7B one-layer W8A16 QDQ attention artifact, compiled as QNN DLC, ran on the exact hosted Snapdragon X Elite CRD (Windows 11 ARM64). Inference returned finite hidden/new-K/new-V tensors bitwise identical to the prior S24/S25 NPU outputs on the same fixture; the profile attributed all 73 execution-detail rows to NPU. Across 100 component samples, nearest-rank p50/p95 was 1.433/1.606 ms; the 1.346 ms estimated-time field is the minimum. Hidden/K/V relative L2 versus original FP32 ONNX CPU was 1.351%/0.377%/0.650%, without a next-token quality tolerance. There is no complete 28-layer prefill/decode/KV-ring/sampling loop, resident Omni controller or complete-request timing. | Qualify a complete X Elite resident artifact/controller with state, token quality, memory admission/cancel and sustained power; [X Elite NPU raw evidence](../../../e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md). | -| PC Snapdragon X Elite CRD / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_ONNX_CPU_VOCODER_PARITY_PROFILED; DIRECTML_RUNTIME_FAILED; TALKER_FP16_NUMERIC_FAIL; QNN_NPU_VOCODER_NUMERIC_MEASURED; PLACEMENT_PENDING; NOT_E2E_PROFILEABLE | The fresh ONNX vocoder inference on the exact Windows 11 ARM64 X Elite CRD requested CPU and returned an unsaturated waveform on the pinned fixture, matching local ONNX Runtime CPU with 8.09e-7 relative L2 and 121.84 dB SNR. Its 94-sample component profile had nearest-rank p50/p95 6.384/6.644 s and all 634 execution-detail rows on CPU; 5.917 s estimated time is the sample minimum. This is one component, without a listening-quality gate. Corrected DirectML inference and profile both failed at provider initialization with Windows error 80070057. A separate FP16 QNN DLC vocoder inference returned an unsaturated waveform at 1.138% relative L2 / 38.88 dB SNR versus local ONNX CPU on the same fixture; NPU node-placement profile is pending and no listening-quality tolerance is set. The 28-layer FP16 QNN talker step executed, yet changed CPU top-1 token 80 to 1720 with 8.57% logits relative L2 on a synthetic cache fixture; a separate ONNX CPU talker compile passed, with inference pending. The older TFLite vocoder target was rejected on Windows ARM; no complete stream exists. | Audit QNN vocoder placement and CPU talker control, investigate DirectML failure, then qualify complete local streaming, memory and sustained behavior; [vocoder](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) and [talker](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) evidence. | +| PC Snapdragon X Elite CRD / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_ONNX_CPU_VOCODER_PARITY_PROFILED; DIRECTML_RUNTIME_FAILED; TALKER_FP16_NUMERIC_FAIL; QNN_NPU_VOCODER_NUMERIC_MEASURED; TALKER_ONNX_CPU_NUMERIC_MEASURED; PLACEMENT_PENDING; NOT_E2E_PROFILEABLE | The fresh ONNX vocoder inference on the exact Windows 11 ARM64 X Elite CRD requested CPU and returned an unsaturated waveform on the pinned fixture, matching local ONNX Runtime CPU with 8.09e-7 relative L2 and 121.84 dB SNR. Its 94-sample component profile had nearest-rank p50/p95 6.384/6.644 s and all 634 execution-detail rows on CPU; 5.917 s estimated time is the sample minimum. This is one component, without a listening-quality gate. Corrected DirectML inference and profile both failed at provider initialization with Windows error 80070057. A separate FP16 QNN DLC vocoder inference returned an unsaturated waveform at 1.138% relative L2 / 38.88 dB SNR versus local ONNX CPU on the same fixture; NPU node-placement profile is pending and no listening-quality tolerance is set. The 28-layer FP16 QNN talker step executed, yet changed CPU top-1 token 80 to 1720 with 8.57% logits relative L2 on a synthetic cache fixture; a separate ONNX CPU-requested talker step preserved CPU token 80 with logits relative L2 2.07e-5 and maximum K/V relative L2 2.18e-5; its placement profile is pending. The older TFLite vocoder target was rejected on Windows ARM; no complete stream exists. | Audit QNN vocoder and CPU talker placement, investigate DirectML failure, then qualify complete local streaming, memory and sustained behavior; [vocoder](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) and [talker](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) evidence. | | PC Snapdragon X Elite CRD / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | PC Snapdragon X Elite CRD / AI Hub | MiniCPM-o 4.5 | COMPONENT_SPEECH_HEAD_NUMERIC_MEASURED; PLACEMENT_PENDING; NOT_E2E_PROFILEABLE | The real 20-layer MiniCPM-o speech-head source compiled to FP16 QNN DLC for the exact X Elite CRD. Same-fixture NPU-requested inference returned 41 finite tensors, preserved ONNX CPU top-1 speech token 1867, and had 0.3170% logits relative L2 and 1.221% maximum K/V relative L2. Actual NPU node assignment and timing await its placement profile. Thinker/encoders/vocoder and a device-local stateful Omni backend remain unqualified. | Audit the [X Elite speech-head placement profile](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md), then validate the complete multimodal pipeline, state, quality, memory and sustained behavior. | | PC Snapdragon X Elite CRD / AI Hub | InternVLA-A1 | COMPONENT_COSMOS_QNN_COMPILE_PASS; INFERENCE_PENDING; NOT_E2E_PROFILEABLE | The real Place_Markpen Cosmos image encoder compiled to FP16 QNN DLC for exact X Elite CRD; same-fixture NPU-requested inference is pending. No encoder numerical result, actual placement, complete action policy or local stateful Omni backend exists. | Audit the [X Elite Cosmos encoder inference](../../../e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md), then validate real observations, action units/timing, state, admission, cancellation and sustained behavior. | @@ -109,7 +109,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i | Embedded RB3 Gen 2 / QCS6490 / AI Hub | Spark-X2.5 | COMPONENT_CPU_PARITY_PASS; COMPONENT_NPU_NUMERIC_UNQUALIFIED; NOT_E2E_PROFILEABLE | On the exact hosted Dragonwing RB3 Gen 2 Vision Kit (QCS6490, QC Linux 1.6), the reused S24 W8A16 QNN DLC and a separately RB3-compiled W8A16 DLC each failed inference/profile at QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR. An FP32 ONNX one-layer CPU route passed with all 56 profile rows on CPU, source-relative hidden/K/V L2 of 8.73e-7/2.98e-7/3.37e-7 and 100-sample component p50/p95 19.304/21.113 ms. A calibrated W8A8 QNN DLC ran with all 61 rows on NPU and p50/p95 5.610/5.761 ms, but hidden/K/V relative L2 was 20.914%/1.702%/4.413% versus FP32 CPU, without a next-token quality gate. These are different precisions, not a qualified speedup; no complete 28-layer prefill/decode/KV-ring/sampling, local Omni controller or complete-request timing exists. | Improve representative W8A8 calibration or find another compatible quality-passing artifact; validate token parity, then full device-resident generation, admission/cancel and sustained power; [RB3 raw evidence](../../../e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md). | | Embedded RB3 Gen 2 / QCS6490 / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_MIXED_VOCODER_PROFILED_NUMERIC_UNQUALIFIED; NOT_E2E_PROFILEABLE | Fresh same-fixture TFLite GPU-requested vocoder inference returned a finite unsaturated waveform, but relative L2 versus original ONNX CPU was 15.73% (16.06 dB SNR), without a listening-quality tolerance; it is not a qualified audio path. The fresh 100-sample component profile had nearest-rank p50/p95 3.099/3.133 s and 403 CPU plus 335 GPU execution-detail rows. Its 3.070 s estimated-time field is the sample minimum, not median. The historical NPU talker build was rejected for floating-point inputs; no full model pipeline exists. | Establish waveform quality and a compatible integer talker artifact, then validate complete local execution, memory and sustained behavior; [vocoder evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md). | | Embedded RB3 Gen 2 / QCS6490 / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | -| Embedded RB3 Gen 2 / QCS6490 / AI Hub | MiniCPM-o 4.5 | COMPONENT_SPEECH_HEAD_FP16_QNN_LOAD_FAILED; CPU_FALLBACK_COMPILE_PENDING; NOT_E2E_PROFILEABLE | The real 20-layer MiniCPM-o speech-head FP16 QNN DLC compiled for exact RB3 Gen 2, but same-fixture NPU-requested inference failed at graph load with QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR. RB3 device attributes do not advertise FP16 HTP support. An explicit TFLite CPU fallback compile is pending; no numerical output, complete model bundle or local stateful controller exists. | Audit the [RB3 speech-head CPU fallback](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md), then qualify the complete multimodal pipeline and sustained behavior. | +| Embedded RB3 Gen 2 / QCS6490 / AI Hub | MiniCPM-o 4.5 | COMPONENT_SPEECH_HEAD_FP16_QNN_LOAD_FAILED; CPU_FALLBACK_COMPILE_PASS_INFERENCE_PENDING; NOT_E2E_PROFILEABLE | The real 20-layer MiniCPM-o speech-head FP16 QNN DLC compiled for exact RB3 Gen 2, but same-fixture NPU-requested inference failed at graph load with QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR. RB3 device attributes do not advertise FP16 HTP support. An explicit TFLite CPU fallback compile passed with same-fixture CPU-requested inference pending; no numerical output, complete model bundle or local stateful controller exists. | Audit the [RB3 speech-head CPU fallback](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md), then qualify the complete multimodal pipeline and sustained behavior. | | Embedded RB3 Gen 2 / QCS6490 / AI Hub | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | ## Measured request timing diff --git a/docs/design/omni_edge_e2e_roadmap.md b/docs/design/omni_edge_e2e_roadmap.md index 11b55ccf0..e94a4db74 100644 --- a/docs/design/omni_edge_e2e_roadmap.md +++ b/docs/design/omni_edge_e2e_roadmap.md @@ -19,9 +19,9 @@ The [Qwen3-TTS Code2Wav Qualcomm follow-up](../../benchmarks/edge_harness/result The identical SA/RB3 TFLite vocoder binary also matched original ONNX CPU within 3.45e-6 relative L2 on a separate local HX370 x86-64 LiteRT 2.2.0 CPU control. Exact-SA CPU-requested parity narrows the failure to the GPU-requested route. The FP32-preserving GPU option did not fix it, so per-node placement and delegate-operation isolation remain open. -A separate [28-layer talker decode-step attempt](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) retains a synthetic fixed-cache fixture and CPU reference. Historical FP16 QNN context-binary inference executed on S24, S25, SA8775P and X Elite, but all four changed the CPU top-1 speech token with 8.57–9.43% logits relative L2. RB3 rejected that floating-point-input artifact. Explicit X Elite ONNX CPU and SA8775P TFLite CPU fallback compiles passed, with same-fixture inference pending. None advances M2 to numerical talker or full-stream qualification yet. +A separate [28-layer talker decode-step attempt](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) retains a synthetic fixed-cache fixture and CPU reference. Historical FP16 QNN context-binary inference executed on S24, S25, SA8775P and X Elite, but all four changed the CPU top-1 speech token with 8.57–9.43% logits relative L2. RB3 rejected that floating-point-input artifact. Explicit X Elite ONNX CPU and SA8775P TFLite CPU fallback compiles passed. X Elite CPU-requested inference preserved CPU top-1 token 80 with logits relative L2 2.07e-5; its placement profile and SA inference remain pending. None advances M2 to numerical talker or full-stream qualification yet. -For M4, a [MiniCPM-o 4.5 speech-head study](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md) recovered the historical 20-layer ONNX source and matching S24/S25 fixtures. Both FP16 context-binary outputs preserved the ONNX CPU top-1 speech token on one decode step, with logits relative L2 0.346%/0.360%. New FP16 QNN DLC compiles passed for SA8775P, X Elite and RB3. SA8775P same-fixture NPU-requested inference preserved CPU top-1 speech token 1867 with 0.3568% logits relative L2 and 1.225% maximum K/V relative L2; placement profiling is pending. X Elite same-fixture inference also preserved token 1867 with 0.3170% logits relative L2; both placement profiles remain pending. RB3 FP16 QNN inference failed at graph load despite compile success; a TFLite CPU fallback compile is pending. This is component C numerical evidence only, not thinker/encoder/vocoder or complete multimodal support. +For M4, a [MiniCPM-o 4.5 speech-head study](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md) recovered the historical 20-layer ONNX source and matching S24/S25 fixtures. Both FP16 context-binary outputs preserved the ONNX CPU top-1 speech token on one decode step, with logits relative L2 0.346%/0.360%. New FP16 QNN DLC compiles passed for SA8775P, X Elite and RB3. SA8775P same-fixture NPU-requested inference preserved CPU top-1 speech token 1867 with 0.3568% logits relative L2 and 1.225% maximum K/V relative L2; placement profiling is pending. X Elite same-fixture inference also preserved token 1867 with 0.3170% logits relative L2; both placement profiles remain pending. RB3 FP16 QNN inference failed at graph load despite compile success; a TFLite CPU fallback compile passed with same-fixture CPU-requested inference pending. This is component C numerical evidence only, not thinker/encoder/vocoder or complete multimodal support. For M4, a [Galaxy S25 Cosmos encoder attempt](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md) repackages the real Place_Markpen FP32 ONNX export after retaining two CPU parity fixtures. FLOAT16 QNN DLC inference returned a latent within 0.0524% relative L2 of the source on one synthetic pattern. Injecting that output into the real policy on local CPU changed synthetic actions by 0.742% relative L2, without a task tolerance. S25 NPU placement profile was pending, and the policy did not run on S25. The same encoder compiled to FP16 QNN DLC for X Elite CRD, with same-fixture NPU-requested inference pending. This is component/action-sensitivity evidence, not device-local M4 qualification.