diff --git a/benchmarks/edge_harness/experiments/audit_spark_qualcomm_attention.py b/benchmarks/edge_harness/experiments/audit_spark_qualcomm_attention.py new file mode 100644 index 000000000..2a5539bc7 --- /dev/null +++ b/benchmarks/edge_harness/experiments/audit_spark_qualcomm_attention.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Audit a real Spark attention-layer AI Hub result without claiming model E2E.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from collections import Counter +from pathlib import Path + +import numpy as np + + +OUTPUT_NAMES = ("hidden", "k_new_0", "v_new_0") +SOURCE_ONNX_SHA256 = "4e2b1c8d791b1ee0e9b75f3322a20a743c0bf8c40f9b98edd963f2944d6300bb" +QUANTIZED_ZIP_SHA256 = "703821ca8ace5672228919ea2add4901def4887748c381b12ffc8b259665c553" +FIXTURE_SHA256 = "5a439e6c6d148f1720938770daa110039a39edde014f1c1592c823c7d7dfc8ed" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(8 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def read_outputs(path: Path, *, hub: bool) -> dict[str, np.ndarray]: + with np.load(path, allow_pickle=False) as data: + actual_names = tuple(data.files) + expected_names = tuple(f"output_{i}__0" for i in range(3)) if hub else OUTPUT_NAMES + if actual_names != expected_names: + raise ValueError(f"{path}: output names/order differ: {actual_names}") + values = {name: np.asarray(data[key]) for name, key in zip(OUTPUT_NAMES, expected_names)} + shapes = ((1, 1, 2048), (1, 2, 1, 256), (1, 2, 1, 256)) + for name, shape in zip(OUTPUT_NAMES, shapes): + value = values[name] + if value.shape != shape or value.dtype != np.float32 or not np.isfinite(value).all(): + raise ValueError(f"{path}: {name} violates FP32 finite layer-output contract") + return values + + +def compare(reference: dict[str, np.ndarray], actual: dict[str, np.ndarray]) -> dict: + result = {} + for name in OUTPUT_NAMES: + left = reference[name].astype(np.float64).ravel() + right = actual[name].astype(np.float64).ravel() + if not np.linalg.norm(left): + raise ValueError(f"{name}: zero reference cannot define relative L2") + result[name] = { + "relative_l2": float(np.linalg.norm(left - right) / np.linalg.norm(left)), + "max_abs": float(np.max(np.abs(left - right))), + "cosine": float(np.dot(left, right) / (np.linalg.norm(left) * np.linalg.norm(right))), + } + return result + + +def nearest_rank(values: list[int], fraction: float) -> int: + ordered = sorted(values) + return ordered[math.ceil(fraction * len(ordered)) - 1] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--device-name", required=True) + for name in ( + "source-onnx", "quantized-zip", "fixture", "source-cpu-output", + "quantized-cpu-output", "s25-output", "s25-inference-report", "device-output", "compile-report", + "inference-report", "profile-report", "output-report", + ): + parser.add_argument(f"--{name}", type=Path, required=True) + args = parser.parse_args() + compile_report = json.loads(args.compile_report.read_text(encoding="utf-8")) + inference = json.loads(args.inference_report.read_text(encoding="utf-8")) + s25_inference = json.loads(args.s25_inference_report.read_text(encoding="utf-8")) + profile = json.loads(args.profile_report.read_text(encoding="utf-8")) + if compile_report.get("status") != "SUCCESS" or inference.get("status") != "SUCCESS" or profile.get("status") != "SUCCESS": + raise RuntimeError("QNN DLC compile, device inference and profile must all succeed") + if (compile_report.get("source_quantized_model_id") != "mmd5e3v3q" + or compile_report.get("options") != "--target_runtime qnn_dlc --quantize_io"): + raise RuntimeError("compile did not use the pinned quantized Spark layer and QNN DLC layout") + target = compile_report["target_model_id"] + if inference["model_id"] != target or profile["model_id"] != target: + raise RuntimeError("device inference/profile used a different compiled artifact") + if inference["input_dataset_id"] != "d7m83gjy2": + raise RuntimeError("device inference did not use the pinned cross-device fixture") + if (s25_inference.get("status") != "SUCCESS" or s25_inference.get("model_id") != "mnowl6xpq" + or s25_inference.get("input_dataset_id") != inference["input_dataset_id"] + or s25_inference.get("device", {}).get("name") != "Samsung Galaxy S25"): + raise RuntimeError("historical S25 output lacks matching model/device/dataset provenance") + if inference["device"]["name"] != args.device_name or profile["device"]["name"] != args.device_name: + raise RuntimeError("jobs did not execute on the exact requested device") + if inference["options"] != "--compute_unit npu" or profile["options"] != "--compute_unit npu": + raise RuntimeError("jobs did not request NPU placement") + + execution = profile["profile"] + times_us = execution["execution_summary"]["all_inference_times"] + if not times_us or any(not isinstance(value, int) or value <= 0 for value in times_us): + raise RuntimeError("missing or invalid device samples") + units = Counter(row.get("compute_unit") for row in execution["execution_detail"]) + if units["NPU"] <= 0: + raise RuntimeError("profile has no attributed NPU execution rows") + + source = read_outputs(args.source_cpu_output, hub=False) + quantized = read_outputs(args.quantized_cpu_output, hub=False) + s25 = read_outputs(args.s25_output, hub=True) + device_output = read_outputs(args.device_output, hub=True) + files = { + name: {"bytes": path.stat().st_size, "sha256": sha256(path)} + for name, path in { + "source_onnx": args.source_onnx, + "quantized_onnx_zip": args.quantized_zip, + "fixture": args.fixture, + "source_cpu_output": args.source_cpu_output, + "quantized_cpu_output": args.quantized_cpu_output, + "s25_output": args.s25_output, + "device_output": args.device_output, + }.items() + } + for name, expected in { + "source_onnx": SOURCE_ONNX_SHA256, + "quantized_onnx_zip": QUANTIZED_ZIP_SHA256, + "fixture": FIXTURE_SHA256, + }.items(): + if files[name]["sha256"] != expected: + raise RuntimeError(f"{name} hash differs from pinned layer artifact or fixture") + report = { + "scope": "one real Spark-X2.5 full-attention decoder layer at one static 1024-token cache shape; no generation E2E", + "status": "component_executed_on_npu_quality_unqualified", + "device": profile["device"], + "compile_job_id": compile_report["job_id"], + "inference_job_id": inference["job_id"], + "profile_job_id": profile["job_id"], + "target_model_id": target, + "input_dataset_id": inference["input_dataset_id"], + "s25_prior_inference_job_id": s25_inference["job_id"], + "files": files, + "source_vs_quantized_cpu": compare(source, quantized), + "quantized_cpu_vs_device_npu": compare(quantized, device_output), + "source_cpu_vs_device_npu": compare(source, device_output), + "s25_npu_vs_device_npu": compare(s25, device_output), + "profile": { + "sample_count": len(times_us), + "sample_min_us": min(times_us), + "nearest_rank_p50_us": nearest_rank(times_us, 0.50), + "nearest_rank_p95_us": nearest_rank(times_us, 0.95), + "reported_estimated_inference_time_us": execution["execution_summary"]["estimated_inference_time"], + "reported_peak_memory_bytes": execution["execution_summary"]["estimated_inference_peak_memory"], + "compute_unit_row_counts": dict(units), + }, + "limits": [ + "No task-level numerical tolerance or next-token quality gate is established.", + "Hub component jobs do not prove device-local prefill, KV/ring update, sampling, continuous generation or Omni deployment.", + "Profile sample latency is one component, not complete-request or sustained thermal performance.", + ], + } + args.output_report.parent.mkdir(parents=True, exist_ok=True) + args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"status": report["status"], "profile": report["profile"]}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/edge_harness/experiments/audit_spark_qualcomm_fallback.py b/benchmarks/edge_harness/experiments/audit_spark_qualcomm_fallback.py new file mode 100644 index 000000000..003de2694 --- /dev/null +++ b/benchmarks/edge_harness/experiments/audit_spark_qualcomm_fallback.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Audit an RB3 Spark attention-layer fallback against the FP32 source.""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter +from pathlib import Path + +from audit_spark_qualcomm_attention import ( + FIXTURE_SHA256, + SOURCE_ONNX_SHA256, + compare, + nearest_rank, + read_outputs, + sha256, +) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--device-name", default="Dragonwing RB3 Gen 2 Vision Kit") + parser.add_argument("--compute-unit", choices=("cpu", "npu"), required=True) + parser.add_argument("--compile-options", required=True) + for name in ( + "source-onnx", "fixture", "source-cpu-output", "compile-report", + "inference-report", "profile-report", "device-output", "output-report", + ): + parser.add_argument(f"--{name}", type=Path, required=True) + args = parser.parse_args() + compile_report = json.loads(args.compile_report.read_text(encoding="utf-8")) + inference = json.loads(args.inference_report.read_text(encoding="utf-8")) + profile = json.loads(args.profile_report.read_text(encoding="utf-8")) + if any(report.get("status") != "SUCCESS" for report in (compile_report, inference, profile)): + raise RuntimeError("compile, inference and profile must all succeed") + if compile_report.get("source_model_id") != "mn14dw6zn" or compile_report.get("options") != args.compile_options: + raise RuntimeError("compile did not use the pinned FP32 Spark source and requested route") + target_id = compile_report["target_model_id"] + if inference.get("model_id") != target_id or profile.get("model_id") != target_id: + raise RuntimeError("inference/profile did not use the compiled artifact") + if inference.get("input_dataset_id") != "d7m83gjy2": + raise RuntimeError("inference did not use the pinned cross-device fixture") + for report in (compile_report, inference, profile): + if report.get("device", {}).get("name") != args.device_name: + raise RuntimeError("job did not use the exact requested device") + requested = f"--compute_unit {args.compute_unit}" + if inference.get("options") != requested or profile.get("options") != requested: + raise RuntimeError("inference/profile requested a different compute unit") + files = { + name: {"bytes": path.stat().st_size, "sha256": sha256(path)} + for name, path in { + "source_onnx": args.source_onnx, + "fixture": args.fixture, + "source_cpu_output": args.source_cpu_output, + "device_output": args.device_output, + }.items() + } + if files["source_onnx"]["sha256"] != SOURCE_ONNX_SHA256 or files["fixture"]["sha256"] != FIXTURE_SHA256: + raise RuntimeError("source or fixture differs from the pinned experiment") + output = read_outputs(args.device_output, hub=True) + source = read_outputs(args.source_cpu_output, hub=False) + execution = profile["profile"] + times_us = execution["execution_summary"]["all_inference_times"] + if not times_us or any(not isinstance(value, int) or value <= 0 for value in times_us): + raise RuntimeError("missing or invalid device samples") + units = Counter(row.get("compute_unit") for row in execution["execution_detail"]) + if units[args.compute_unit.upper()] <= 0: + raise RuntimeError("profile has no rows attributed to the requested compute unit") + report = { + "scope": "one real Spark-X2.5 attention layer at one static 1024-token cache shape; no generation E2E", + "status": f"component_executed_on_{args.compute_unit}_quality_unqualified", + "device": profile["device"], + "compile_job_id": compile_report["job_id"], + "inference_job_id": inference["job_id"], + "profile_job_id": profile["job_id"], + "target_model_id": target_id, + "input_dataset_id": inference["input_dataset_id"], + "compile_options": args.compile_options, + "files": files, + "source_cpu_vs_device": compare(source, output), + "profile": { + "sample_count": len(times_us), + "sample_min_us": min(times_us), + "nearest_rank_p50_us": nearest_rank(times_us, 0.50), + "nearest_rank_p95_us": nearest_rank(times_us, 0.95), + "reported_estimated_inference_time_us": execution["execution_summary"]["estimated_inference_time"], + "reported_peak_memory_bytes": execution["execution_summary"]["estimated_inference_peak_memory"], + "compute_unit_row_counts": dict(units), + }, + "limits": [ + "No next-token or task-level numerical tolerance is established.", + "No complete prefill, KV/ring, sampling, continuous generation or Omni deployment is tested.", + "Hosted component latency is not a complete request or sustained thermal profile.", + ], + } + args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"status": report["status"], "profile": report["profile"]}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/edge_harness/experiments/collect_spark_aihub_component.py b/benchmarks/edge_harness/experiments/collect_spark_aihub_component.py new file mode 100644 index 000000000..dd0147ad7 --- /dev/null +++ b/benchmarks/edge_harness/experiments/collect_spark_aihub_component.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Preserve raw Workbench Spark-layer jobs submitted in an evidence directory. + +Requires qai-hub authentication through the local client configuration. The +submission JSON files identify existing jobs; this script never resubmits them. +""" + +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np +import qai_hub as hub + + +def device_record(device: object) -> dict: + return { + "name": device.name, + "os": device.os, + "attributes": list(device.attributes), + } + + +def collect(evidence_dir: Path, kind: str, client: hub.Client) -> str: + submission = json.loads((evidence_dir / f"{kind}_submission.json").read_text(encoding="utf-8")) + report_path = evidence_dir / f"{kind}_report.json" + if report_path.exists(): + existing = json.loads(report_path.read_text(encoding="utf-8")) + if existing.get("job_id") != submission["job_id"]: + raise RuntimeError(f"{report_path}: job ID differs from its submission") + return f"{kind}: {submission['job_id']} {existing['status']} (already collected)" + job = client.get_job(submission["job_id"]) + status = job.get_status() + if status.code not in {"SUCCESS", "FAILED", "CANCELED"}: + return f"{kind}: {submission['job_id']} {status.code} (pending)" + report = { + "timestamp_utc": datetime.now(timezone.utc).isoformat(), + "qai_hub_version": str(hub.__version__), + "facility": "Qualcomm AI Hub Workbench", + "job_id": submission["job_id"], + "url": submission["url"], + "status": status.code, + "status_message": status.message or "", + "options": job.options, + "hub_version": str(job.hub_version), + "device": device_record(job.device), + "model_id": job.model.model_id, + } + if kind == "inference": + report["input_dataset_id"] = job.inputs.dataset_id + if status.code == "SUCCESS": + output_path = evidence_dir / "device_output.npz" + outputs = job.download_output_data() + arrays = {f"{name}__{i}": value for name, values in outputs.items() for i, value in enumerate(values)} + np.savez_compressed(output_path, **arrays) + report["outputs"] = output_path.name + elif kind == "profile": + report["shapes"] = job.shapes + if status.code == "SUCCESS": + report["profile"] = job.download_profile() + else: + raise ValueError(f"unexpected job kind {kind}") + report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + return f"{kind}: {submission['job_id']} {status.code}" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("evidence_dirs", nargs="+", type=Path) + args = parser.parse_args() + client = hub.Client() + for directory in args.evidence_dirs: + for kind in ("inference", "profile"): + print(f"{directory.name}: {collect(directory, kind, client)}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md index 15f30768a..c79a2d59f 100644 --- a/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md @@ -20,6 +20,10 @@ The [native-storage follow-up](evidence/minicpmo_omni_native_weights_io/README.m A separate [WSL CPU Spark GGUF Omni profile](evidence/spark_omni_wsl_cpu/README.md) ran the same pinned Q4_K_M artifact through a Linux CPU-only llama.cpp server and current Omni controller. It passed one warmup and 20/20 serial inventory requests at nearest-rank complete-request wall p50/p95 **7.82/7.97 s**, a public `AsyncOmni.generate` text request, explicit context refusal and in-flight cancellation cleanup with a 4 GiB host reservation. The Linux server omitted the Windows offload-count line; the stage now requires positive CPU tensor-buffer evidence for that log layout and rejects accelerator buffers. This is independent of the earlier WSL vLLM INT8 result and does not qualify incremental streaming or general text quality. +The [hosted Galaxy S24 Spark experiment](evidence/spark_s24_full_attention/README.md) moved one real full-attention decoder layer with a fixed 1,024-token cache shape to the phone NPU. Its W8A16 QDQ QNN DLC inference matched the earlier S25 NPU outputs exactly on one pinned dataset; 100 NPU-attributed component samples had nearest-rank p50/p95 **1.109/1.205 ms**. Separate local FP32 and quantized CPU runs measured the numerical change, with 1.351% hidden-state relative L2 from original FP32 source to S24 output and no task-quality tolerance. AI Hub Workbench runs components on hosted physical devices, not the stateful local Spark generation loop; this cell remains NOT E2E. + +The same layer artifact then ran on hosted [Snapdragon X Elite CRD](evidence/spark_xelite_full_attention/README.md) and [SA8775P ADP](evidence/spark_sa8775_full_attention/README.md) NPUs, with exact same-fixture outputs as S24/S25 and 100-sample component p50/p95 **1.433/1.606 ms** and **1.837/2.079 ms**, respectively. On [RB3 Gen 2 / QCS6490](evidence/spark_rb3_full_attention/README.md), both the reused and exact-device-compiled W8A16 QNN DLCs failed at graph composition. An FP32 ONNX CPU layer passed with source-relative L2 below 9e-7; calibrated W8A8 ran on NPU but had 20.9% hidden-state relative L2 and is numerically unqualified. These are separate device components, not full-model deployment or cross-device speed comparisons. + | Device / model | Verified depth | Observed output and time | Remaining gate | |---|---|---|---| | Native Windows HX370 Radeon 890M / InternVLA-A1 Cosmos CI8x8 encoder | Component C; unchanged Omni encoder source in separate DirectML/PyTorch 2.4 environment, not an Omni full-policy stage | One warmup + 20 serial patterned 256×256 image encodes, including upload/readback: Radeon p50/p95 26.56/27.49 ms versus CPU 156.49/158.71 ms; relative L2 error 4.73e-07. Encoder weights and output device checked. | Complete policy/action and real-observation parity; policy/controller runtime compatibility, bounded handoff, shared-RAM admission and end-to-end benefit. [Raw evidence](evidence/internvla_cosmos_radeon890m/README.md). | diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md new file mode 100644 index 000000000..eea900b3a --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md @@ -0,0 +1,18 @@ +# Spark-X2.5 attention layer on Dragonwing RB3 Gen 2 / QCS6490 + +**The hosted RB3 kit (QC Linux 1.6, Hexagon v68) ran one real Spark-X2.5-1.7B attention layer on CPU with near-FP32 numerical parity. A separate W8A8 NPU variant executed but has 20.9% hidden-state relative L2 error on the pinned fixture. Neither variant qualifies full Spark generation.** The [S24 provenance](../spark_s24_full_attention/README.md) pins the FP32 source ONNX, its W8A16 QDQ form, calibration and exact 1,024-token-cache input. The [artifact manifest](artifact_manifest.json) pins downloaded compile outputs outside Git. + +Four routes were tested on exact device **Dragonwing RB3 Gen 2 Vision Kit**, using AI Hub Workbench client 0.55.0 and service build in each raw job report: + +| Route | Jobs and outcome | One-layer evidence | +|---|---|---| +| Reuse S24-built W8A16 QNN DLC `mq9y70k0n` | [Inference](inference_report.json) `j57e4k4lp` and [profile](profile_report.json) `jp4y1m1vp` both failed: `QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR`. | No outputs or timing samples. | +| Compile same QDQ source for RB3 as W8A16 QNN DLC `mn0g3ye8m` | [Compile](compile_rb3_report.json) `jgnznozrg` passed. Targeted [inference](targeted/inference_report.json) `jglyxoy85` and [profile](targeted/profile_report.json) `j5687r80g` both failed with the same graph error. | Device-specific recompilation did not resolve this graph/runtime failure. It does not establish that all W8A16 Spark exports are unsupported. | +| Compile original FP32 source to ONNX `mmxj02ljq`, request CPU | [Compile](compile_cpu_report.json) `jglyxowj5`, [inference](cpu/inference_report.json) `jgolrodqg` and [profile](cpu/profile_report.json) `jpvldemk5` passed. All 56 execution-detail rows were CPU. | [Output](cpu/device_output.npz) hidden/K/V relative L2 versus original ONNX Runtime 1.29 CPU was 8.73e-7/2.98e-7/3.37e-7. [Audit](cpu/audit_report.json): 100 samples, nearest-rank p50/p95 19.304/21.113 ms (minimum 19.104 ms). This is explicitly CPU, not NPU fallback masquerading as acceleration. | +| Compile original source with calibration dataset `d7x8e3d59` using `--quantize_full_type int8 --quantize_io` to QNN DLC `mqkp10ezm`, request NPU | [Compile](compile_w8a8_report.json) `jpyoxq885`, [inference](w8a8/inference_report.json) `jgzlm8yo5` and [profile](w8a8/profile_report.json) `j5wl78z3p` passed. All 61 execution-detail rows were NPU. | [Output](w8a8/device_output.npz) hidden/K/V relative L2 versus FP32 CPU was **20.914%/1.702%/4.413%**. [Audit](w8a8/audit_report.json): 100 samples, p50/p95 5.610/5.761 ms (minimum 5.089 ms). This precision is **numerically unqualified**; no next-token/task tolerance is established. | + +All successful jobs used the same fixture dataset `d7m83gjy2` and returned finite FP32 hidden/new-key/new-value tensors. The service `estimated_inference_time` equals each sample minimum, not the median. The [CPU](cpu/audit_report.json) and [W8A8 NPU](w8a8/audit_report.json) audits pin source and fixture hashes, exact job/device/model IDs, output hashes, placement rows and full precision differences. [Submission records](compile_w8a8_submission.json), [initial failed submissions](inference_submission.json), [RB3 W8A16 compile submission](compile_rb3_submission.json), and [CPU compile submission](compile_cpu_submission.json) preserve attempted settings. The scripts [collect existing jobs](../../../../experiments/collect_spark_aihub_component.py) and [audit fallback variants](../../../../experiments/audit_spark_qualcomm_fallback.py) without embedded credentials. + +The CPU and NPU numbers are different precisions and outputs, not a qualified speedup comparison. Neither job includes 27 other decoder layers, prefill, continuous decode, KV/ring, sampling, a resident Omni controller, complete-request timing or sustained power/thermal behavior. Workbench is a test facility, not a deployment dependency. + +**Matrix status: CPU component numerical pass; W8A8 NPU component execution with unresolved numerical quality; NOT E2E.** The next NPU experiment needs representative multi-prompt calibration and layer/next-token parity gates, then a compatible full-model artifact. The full embedded route also needs device-local state, memory admission, cancellation and sustained profiling. The two W8A16 graph errors remain specific to the tested QDQ export and SDK/device combination. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/artifact_manifest.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/artifact_manifest.json new file mode 100644 index 000000000..5919f1480 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/artifact_manifest.json @@ -0,0 +1,33 @@ +{ + "scope": "Pinned Spark attention-layer variants for RB3 remediation; binaries retained outside Git", + "artifacts": { + "s24_compiled_w8a16_dlc": { + "model_id": "mq9y70k0n", + "compile_job_id": "jpe7z8l15", + "downloaded_path": "/home/zhout/project/edge_infer/models/spark_s24_qnn_dlc_20260924.dlc", + "bytes": 52036972, + "sha256": "8f1518f1f392434306dba11fd7115fe7629657ab7bfc834c0f78ee6b684e5154" + }, + "rb3_targeted_w8a16_dlc": { + "model_id": "mn0g3ye8m", + "compile_job_id": "jgnznozrg", + "downloaded_path": "/home/zhout/project/edge_infer/models/spark_rb3_qnn_dlc_20260924.dlc", + "bytes": 52036972, + "sha256": "614ea0b07d9119581c563d9479181390fb007efb39b538381b971ce12ef39161" + }, + "rb3_w8a8_dlc": { + "model_id": "mqkp10ezm", + "compile_job_id": "jpyoxq885", + "downloaded_path": "/home/zhout/project/edge_infer/models/spark_rb3_w8a8_dlc_20260924.dlc", + "bytes": 51452356, + "sha256": "637f4dd52ddf86e53c46c1ae5c1ef3b030b822f866575835aa66e6b86ad3cfd2" + }, + "rb3_fp32_onnx_cpu": { + "model_id": "mmxj02ljq", + "compile_job_id": "jglyxowj5", + "downloaded_path": "/home/zhout/project/edge_infer/models/spark_rb3_onnx_cpu_20260924.onnx.onnx.zip", + "bytes": 107095517, + "sha256": "a64c220f32f7a4a5652594014e39e8ead2f688b7dabf0ed878620b61d94d7ec2" + } + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_cpu_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_cpu_report.json new file mode 100644 index 000000000..6aea2fb8b --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_cpu_report.json @@ -0,0 +1,31 @@ +{ + "timestamp_utc": "2026-09-23T21:56:04.464159+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jglyxowj5", + "url": "https://workbench.aihub.qualcomm.com/jobs/jglyxowj5/", + "status": "SUCCESS", + "status_message": "", + "source_model_id": "mn14dw6zn", + "target_model_id": "mmxj02ljq", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "options": "--target_runtime onnx", + "hub_version": "aihub-2026.09.11.0" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_cpu_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_cpu_submission.json new file mode 100644 index 000000000..fde9bc763 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_cpu_submission.json @@ -0,0 +1,11 @@ +{ + "timestamp_utc": "2026-09-23T21:54:10.264314+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "job_id": "jglyxowj5", + "url": "https://workbench.aihub.qualcomm.com/jobs/jglyxowj5/", + "source_model_id": "mn14dw6zn", + "device": "Dragonwing RB3 Gen 2 Vision Kit", + "options": "--target_runtime onnx", + "reason": "Explicit CPU-only component fallback to separate graph compatibility from NPU capability" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_rb3_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_rb3_report.json new file mode 100644 index 000000000..502b33d72 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_rb3_report.json @@ -0,0 +1,31 @@ +{ + "timestamp_utc": "2026-09-23T21:47:29.642901+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jgnznozrg", + "url": "https://workbench.aihub.qualcomm.com/jobs/jgnznozrg/", + "status": "SUCCESS", + "status_message": "", + "source_quantized_model_id": "mmd5e3v3q", + "target_model_id": "mn0g3ye8m", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "options": "--target_runtime qnn_dlc --quantize_io", + "hub_version": "aihub-2026.09.11.0" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_rb3_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_rb3_submission.json new file mode 100644 index 000000000..5ef16a3da --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_rb3_submission.json @@ -0,0 +1,11 @@ +{ + "timestamp_utc": "2026-09-23T21:45:01.402565+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "job_id": "jgnznozrg", + "url": "https://workbench.aihub.qualcomm.com/jobs/jgnznozrg/", + "source_quantized_model_id": "mmd5e3v3q", + "device": "Dragonwing RB3 Gen 2 Vision Kit", + "options": "--target_runtime qnn_dlc --quantize_io", + "reason": "S24-targeted QNN DLC failed composeGraphsFromDlc on RB3" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_w8a8_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_w8a8_report.json new file mode 100644 index 000000000..f1a5aeabc --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_w8a8_report.json @@ -0,0 +1,32 @@ +{ + "timestamp_utc": "2026-09-23T21:56:39.060765+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jpyoxq885", + "url": "https://workbench.aihub.qualcomm.com/jobs/jpyoxq885/", + "status": "SUCCESS", + "status_message": "", + "source_model_id": "mn14dw6zn", + "calibration_dataset_id": "d7x8e3d59", + "target_model_id": "mqkp10ezm", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "options": "--target_runtime qnn_dlc --quantize_full_type int8 --quantize_io", + "hub_version": "aihub-2026.09.11.0" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_w8a8_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_w8a8_submission.json new file mode 100644 index 000000000..3ba00705e --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/compile_w8a8_submission.json @@ -0,0 +1,12 @@ +{ + "timestamp_utc": "2026-09-23T21:53:55.558562+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "job_id": "jpyoxq885", + "url": "https://workbench.aihub.qualcomm.com/jobs/jpyoxq885/", + "source_model_id": "mn14dw6zn", + "calibration_dataset_id": "d7x8e3d59", + "device": "Dragonwing RB3 Gen 2 Vision Kit", + "options": "--target_runtime qnn_dlc --quantize_full_type int8 --quantize_io", + "reason": "W8A16 QNN DLC failed graph composition on RB3, including exact-device compile" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/audit_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/audit_report.json new file mode 100644 index 000000000..283931336 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/audit_report.json @@ -0,0 +1,79 @@ +{ + "scope": "one real Spark-X2.5 attention layer at one static 1024-token cache shape; no generation E2E", + "status": "component_executed_on_cpu_quality_unqualified", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "compile_job_id": "jglyxowj5", + "inference_job_id": "jgolrodqg", + "profile_job_id": "jpvldemk5", + "target_model_id": "mmxj02ljq", + "input_dataset_id": "d7m83gjy2", + "compile_options": "--target_runtime onnx", + "files": { + "source_onnx": { + "bytes": 205618089, + "sha256": "4e2b1c8d791b1ee0e9b75f3322a20a743c0bf8c40f9b98edd963f2944d6300bb" + }, + "fixture": { + "bytes": 3901238, + "sha256": "5a439e6c6d148f1720938770daa110039a39edde014f1c1592c823c7d7dfc8ed" + }, + "source_cpu_output": { + "bytes": 12103, + "sha256": "6bbfb8e6ccb09475391ce1dbd2e855205720f4367c038a0d5be35e7dfeb2f768" + }, + "device_output": { + "bytes": 12128, + "sha256": "442d3a8fef165584c853daf87592e4dc7ed3bc9788eee93b2fc6c67cbc6770b4" + } + }, + "source_cpu_vs_device": { + "hidden": { + "relative_l2": 8.726789979684891e-07, + "max_abs": 3.2782554626464844e-07, + "cosine": 0.9999999999996264 + }, + "k_new_0": { + "relative_l2": 2.984418086993196e-07, + "max_abs": 2.384185791015625e-06, + "cosine": 0.9999999999999881 + }, + "v_new_0": { + "relative_l2": 3.3724649115031866e-07, + "max_abs": 8.344650268554688e-07, + "cosine": 0.9999999999999739 + } + }, + "profile": { + "sample_count": 100, + "sample_min_us": 19104, + "nearest_rank_p50_us": 19304, + "nearest_rank_p95_us": 21113, + "reported_estimated_inference_time_us": 19104, + "reported_peak_memory_bytes": 227733504, + "compute_unit_row_counts": { + "CPU": 56 + } + }, + "limits": [ + "No next-token or task-level numerical tolerance is established.", + "No complete prefill, KV/ring, sampling, continuous generation or Omni deployment is tested.", + "Hosted component latency is not a complete request or sustained thermal profile." + ] +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/device_output.npz b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/device_output.npz new file mode 100644 index 000000000..f460c7997 Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/device_output.npz differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/inference_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/inference_report.json new file mode 100644 index 000000000..db3e915c8 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/inference_report.json @@ -0,0 +1,32 @@ +{ + "timestamp_utc": "2026-09-23T22:01:39.257777+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jgolrodqg", + "url": "https://workbench.aihub.qualcomm.com/jobs/jgolrodqg/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit cpu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "model_id": "mmxj02ljq", + "input_dataset_id": "d7m83gjy2", + "outputs": "device_output.npz" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/inference_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/inference_submission.json new file mode 100644 index 000000000..01048beab --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/inference_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-23T21:55:49.636711+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jglyxowj5", + "target_model_id": "mmxj02ljq", + "input_dataset_id": "d7m83gjy2", + "device": "Dragonwing RB3 Gen 2 Vision Kit", + "options": "--compute_unit cpu", + "kind": "inference", + "status": "SUBMITTED", + "job_id": "jgolrodqg", + "url": "https://workbench.aihub.qualcomm.com/jobs/jgolrodqg/" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/profile_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/profile_report.json new file mode 100644 index 000000000..3f3d15d33 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/profile_report.json @@ -0,0 +1,632 @@ +{ + "timestamp_utc": "2026-09-23T22:01:44.553558+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jpvldemk5", + "url": "https://workbench.aihub.qualcomm.com/jobs/jpvldemk5/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit cpu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "model_id": "mmxj02ljq", + "shapes": { + "x": [ + [ + 1, + 1, + 2048 + ], + "float32" + ], + "cos_full": [ + [ + 1, + 1, + 1, + 64 + ], + "float32" + ], + "sin_full": [ + [ + 1, + 1, + 1, + 64 + ], + "float32" + ], + "mask_full": [ + [ + 1, + 1, + 1, + 1025 + ], + "float32" + ], + "k_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "float32" + ], + "v_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "float32" + ] + }, + "profile": { + "execution_summary": { + "estimated_inference_time": 19104, + "estimated_inference_peak_memory": 227733504, + "first_load_time": 311945, + "first_load_peak_memory": 221429760, + "warm_load_time": 319951, + "warm_load_peak_memory": 217300992, + "compile_time": 0, + "compile_peak_memory": 0, + "compile_memory_increase_range": [ + 0, + 0 + ], + "compile_memory_peak_range": [ + 0, + 0 + ], + "first_load_memory_increase_range": [ + 10790272, + 11590944 + ], + "first_load_memory_peak_range": [ + 218533888, + 219334560 + ], + "warm_load_memory_increase_range": [ + 6655248, + 7457200 + ], + "warm_load_memory_peak_range": [ + 214401024, + 215202976 + ], + "inference_memory_increase_range": [ + 0, + 0 + ], + "inference_memory_peak_range": [ + 4120576, + 216040736 + ], + "all_compile_times": [], + "all_first_load_times": [ + 311945 + ], + "all_warm_load_times": [ + 319951 + ], + "all_inference_times": [ + 22116, + 19121, + 21897, + 19651, + 21601, + 19307, + 19264, + 21113, + 19333, + 19307, + 21370, + 19104, + 19275, + 19320, + 19207, + 19300, + 19644, + 19384, + 19192, + 19301, + 19300, + 19221, + 19480, + 19284, + 19332, + 19419, + 19308, + 19228, + 19176, + 19409, + 19309, + 19287, + 19256, + 19221, + 19337, + 19359, + 19250, + 19694, + 19316, + 19240, + 19270, + 19262, + 19271, + 19198, + 19311, + 25366, + 19323, + 19277, + 19319, + 19415, + 19243, + 19588, + 19252, + 19318, + 19292, + 19307, + 19237, + 19352, + 19246, + 19242, + 19322, + 19395, + 19347, + 19456, + 19315, + 19256, + 19286, + 19329, + 19337, + 19336, + 19898, + 19727, + 19401, + 19254, + 19255, + 19526, + 19328, + 19306, + 19268, + 19209, + 19263, + 19177, + 19336, + 19241, + 19235, + 19256, + 19428, + 19248, + 19398, + 19584, + 19191, + 19304, + 19218, + 19296, + 19222, + 19225, + 19245, + 19332, + 19245, + 19292 + ] + }, + "execution_detail": [ + { + "name": "node_Mul_0", + "type": "Mul", + "compute_unit": "CPU", + "execution_time": 51, + "execution_cycles": 0 + }, + { + "name": "/layers.0/ReduceMean", + "type": "ReduceMean", + "compute_unit": "CPU", + "execution_time": 22, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add", + "type": "Add", + "compute_unit": "CPU", + "execution_time": 12, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Sqrt", + "type": "Sqrt", + "compute_unit": "CPU", + "execution_time": 13, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Div", + "type": "Div", + "compute_unit": "CPU", + "execution_time": 10, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_1", + "type": "Mul", + "compute_unit": "CPU", + "execution_time": 18, + "execution_cycles": 0 + }, + { + "name": "/layers.0/q_k_v_proj/MatMul", + "type": "MatMul", + "compute_unit": "CPU", + "execution_time": 2100, + "execution_cycles": 0 + }, + { + "name": "node_Split_20", + "type": "Split", + "compute_unit": "CPU", + "execution_time": 81, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape", + "type": "Reshape", + "compute_unit": "CPU", + "execution_time": 42, + "execution_cycles": 0 + }, + { + "name": "fused_/layers.0/Slice_3_/layers.0/Slice_6", + "type": "Slice", + "compute_unit": "CPU", + "execution_time": 68, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Neg", + "type": "Neg", + "compute_unit": "CPU", + "execution_time": 38, + "execution_cycles": 0 + }, + { + "name": "fused_/layers.0/Slice_3_/layers.0/Slice_5", + "type": "Slice", + "compute_unit": "CPU", + "execution_time": 52, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Concat", + "type": "Concat", + "compute_unit": "CPU", + "execution_time": 48, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_3", + "type": "Mul", + "compute_unit": "CPU", + "execution_time": 48, + "execution_cycles": 0 + }, + { + "name": "node_Split_22", + "type": "Split", + "compute_unit": "CPU", + "execution_time": 46, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_2", + "type": "Mul", + "compute_unit": "CPU", + "execution_time": 41, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_1", + "type": "Add", + "compute_unit": "CPU", + "execution_time": 42, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Concat_1", + "type": "Concat", + "compute_unit": "CPU", + "execution_time": 44, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_4", + "type": "Reshape", + "compute_unit": "CPU", + "execution_time": 38, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_1", + "type": "Reshape", + "compute_unit": "CPU", + "execution_time": 36, + "execution_cycles": 0 + }, + { + "name": "fused_/layers.0/Slice_7_/layers.0/Slice_10", + "type": "Slice", + "compute_unit": "CPU", + "execution_time": 50, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Neg_1", + "type": "Neg", + "compute_unit": "CPU", + "execution_time": 37, + "execution_cycles": 0 + }, + { + "name": "fused_/layers.0/Slice_7_/layers.0/Slice_9", + "type": "Slice", + "compute_unit": "CPU", + "execution_time": 46, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Concat_2", + "type": "Concat", + "compute_unit": "CPU", + "execution_time": 43, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_5", + "type": "Mul", + "compute_unit": "CPU", + "execution_time": 44, + "execution_cycles": 0 + }, + { + "name": "node_Split_24", + "type": "Split", + "compute_unit": "CPU", + "execution_time": 45, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_4", + "type": "Mul", + "compute_unit": "CPU", + "execution_time": 38, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_2", + "type": "Add", + "compute_unit": "CPU", + "execution_time": 39, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Concat_3", + "type": "Concat", + "compute_unit": "CPU", + "execution_time": 44, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Concat_4", + "type": "Concat", + "compute_unit": "CPU", + "execution_time": 309, + "execution_cycles": 0 + }, + { + "name": "/layers.0/MatMul/MatmulTransposeFusion//MatMulScaleFusion/", + "type": "FusedMatMul", + "compute_unit": "CPU", + "execution_time": 578, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_3", + "type": "Add", + "compute_unit": "CPU", + "execution_time": 78, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Softmax", + "type": "Softmax", + "compute_unit": "CPU", + "execution_time": 95, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_2", + "type": "Reshape", + "compute_unit": "CPU", + "execution_time": 45, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Concat_5", + "type": "Concat", + "compute_unit": "CPU", + "execution_time": 324, + "execution_cycles": 0 + }, + { + "name": "/layers.0/MatMul_1", + "type": "MatMul", + "compute_unit": "CPU", + "execution_time": 616, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_5", + "type": "Reshape", + "compute_unit": "CPU", + "execution_time": 45, + "execution_cycles": 0 + }, + { + "name": "/layers.0/g_proj/MatMul", + "type": "MatMul", + "compute_unit": "CPU", + "execution_time": 71, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Sigmoid", + "type": "Sigmoid", + "compute_unit": "CPU", + "execution_time": 40, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_3", + "type": "Reshape", + "compute_unit": "CPU", + "execution_time": 36, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_7", + "type": "Mul", + "compute_unit": "CPU", + "execution_time": 51, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_6", + "type": "Reshape", + "compute_unit": "CPU", + "execution_time": 38, + "execution_cycles": 0 + }, + { + "name": "/layers.0/out_proj/MatMul", + "type": "MatMul", + "compute_unit": "CPU", + "execution_time": 1716, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_4", + "type": "Add", + "compute_unit": "CPU", + "execution_time": 64, + "execution_cycles": 0 + }, + { + "name": "node_Mul_1", + "type": "Mul", + "compute_unit": "CPU", + "execution_time": 41, + "execution_cycles": 0 + }, + { + "name": "/layers.0/ReduceMean_1", + "type": "ReduceMean", + "compute_unit": "CPU", + "execution_time": 51, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_5", + "type": "Add", + "compute_unit": "CPU", + "execution_time": 38, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Sqrt_1", + "type": "Sqrt", + "compute_unit": "CPU", + "execution_time": 36, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Div_1", + "type": "Div", + "compute_unit": "CPU", + "execution_time": 43, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_9", + "type": "Mul", + "compute_unit": "CPU", + "execution_time": 52, + "execution_cycles": 0 + }, + { + "name": "/layers.0/up_proj/MatMul", + "type": "MatMul", + "compute_unit": "CPU", + "execution_time": 4437, + "execution_cycles": 0 + }, + { + "name": "/layers.0/gate_proj/MatMul", + "type": "MatMul", + "compute_unit": "CPU", + "execution_time": 4404, + "execution_cycles": 0 + }, + { + "name": "node_Gelu_2", + "type": "Gelu", + "compute_unit": "CPU", + "execution_time": 147, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_12", + "type": "Mul", + "compute_unit": "CPU", + "execution_time": 62, + "execution_cycles": 0 + }, + { + "name": "/layers.0/down_proj/MatMul", + "type": "MatMul", + "compute_unit": "CPU", + "execution_time": 4396, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_7", + "type": "Add", + "compute_unit": "CPU", + "execution_time": 74, + "execution_cycles": 0 + } + ] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/profile_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/profile_submission.json new file mode 100644 index 000000000..1d90e0fd4 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/cpu/profile_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-23T21:55:51.569947+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jglyxowj5", + "target_model_id": "mmxj02ljq", + "input_dataset_id": null, + "device": "Dragonwing RB3 Gen 2 Vision Kit", + "options": "--compute_unit cpu", + "kind": "profile", + "status": "SUBMITTED", + "job_id": "jpvldemk5", + "url": "https://workbench.aihub.qualcomm.com/jobs/jpvldemk5/" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/inference_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/inference_report.json new file mode 100644 index 000000000..2a8a922a9 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/inference_report.json @@ -0,0 +1,31 @@ +{ + "timestamp_utc": "2026-09-23T21:44:28.625840+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "j57e4k4lp", + "url": "https://workbench.aihub.qualcomm.com/jobs/j57e4k4lp/", + "status": "FAILED", + "status_message": "Failed to profile the model: Failed to call QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR.", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "model_id": "mq9y70k0n", + "input_dataset_id": "d7m83gjy2" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/inference_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/inference_submission.json new file mode 100644 index 000000000..d3271a2fb --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/inference_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-23T21:27:00.903338+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jpe7z8l15", + "target_model_id": "mq9y70k0n", + "input_dataset_id": "d7m83gjy2", + "device": "Dragonwing RB3 Gen 2 Vision Kit", + "options": "--compute_unit npu", + "kind": "inference", + "status": "SUBMITTED", + "job_id": "j57e4k4lp", + "url": "https://workbench.aihub.qualcomm.com/jobs/j57e4k4lp/" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/profile_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/profile_report.json new file mode 100644 index 000000000..f22f6b9c4 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/profile_report.json @@ -0,0 +1,85 @@ +{ + "timestamp_utc": "2026-09-23T21:44:29.795793+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jp4y1m1vp", + "url": "https://workbench.aihub.qualcomm.com/jobs/jp4y1m1vp/", + "status": "FAILED", + "status_message": "Failed to profile the model: Failed to call QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR.", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "model_id": "mq9y70k0n", + "shapes": { + "x": [ + [ + 1, + 1, + 2048 + ], + "uint16" + ], + "cos_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint16" + ], + "sin_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint16" + ], + "mask_full": [ + [ + 1, + 1, + 1, + 1025 + ], + "uint16" + ], + "k_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint16" + ], + "v_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint16" + ] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/profile_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/profile_submission.json new file mode 100644 index 000000000..26542841d --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/profile_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-23T21:27:02.068005+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jpe7z8l15", + "target_model_id": "mq9y70k0n", + "input_dataset_id": null, + "device": "Dragonwing RB3 Gen 2 Vision Kit", + "options": "--compute_unit npu", + "kind": "profile", + "status": "SUBMITTED", + "job_id": "jp4y1m1vp", + "url": "https://workbench.aihub.qualcomm.com/jobs/jp4y1m1vp/" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/targeted/inference_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/targeted/inference_report.json new file mode 100644 index 000000000..6567ab226 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/targeted/inference_report.json @@ -0,0 +1,31 @@ +{ + "timestamp_utc": "2026-09-23T21:53:06.828554+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jglyxoy85", + "url": "https://workbench.aihub.qualcomm.com/jobs/jglyxoy85/", + "status": "FAILED", + "status_message": "Failed to profile the model: Failed to call QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR.", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "model_id": "mn0g3ye8m", + "input_dataset_id": "d7m83gjy2" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/targeted/inference_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/targeted/inference_submission.json new file mode 100644 index 000000000..76aee58bc --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/targeted/inference_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-23T21:47:50.296704+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jgnznozrg", + "target_model_id": "mn0g3ye8m", + "input_dataset_id": "d7m83gjy2", + "device": "Dragonwing RB3 Gen 2 Vision Kit", + "options": "--compute_unit npu", + "kind": "inference", + "status": "SUBMITTED", + "job_id": "jglyxoy85", + "url": "https://workbench.aihub.qualcomm.com/jobs/jglyxoy85/" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/targeted/profile_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/targeted/profile_report.json new file mode 100644 index 000000000..c19853944 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/targeted/profile_report.json @@ -0,0 +1,85 @@ +{ + "timestamp_utc": "2026-09-23T21:53:07.997274+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "j5687r80g", + "url": "https://workbench.aihub.qualcomm.com/jobs/j5687r80g/", + "status": "FAILED", + "status_message": "Failed to profile the model: Failed to call QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR.", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "model_id": "mn0g3ye8m", + "shapes": { + "x": [ + [ + 1, + 1, + 2048 + ], + "uint16" + ], + "cos_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint16" + ], + "sin_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint16" + ], + "mask_full": [ + [ + 1, + 1, + 1, + 1025 + ], + "uint16" + ], + "k_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint16" + ], + "v_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint16" + ] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/targeted/profile_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/targeted/profile_submission.json new file mode 100644 index 000000000..3b7899793 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/targeted/profile_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-23T21:47:51.950992+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jgnznozrg", + "target_model_id": "mn0g3ye8m", + "input_dataset_id": null, + "device": "Dragonwing RB3 Gen 2 Vision Kit", + "options": "--compute_unit npu", + "kind": "profile", + "status": "SUBMITTED", + "job_id": "j5687r80g", + "url": "https://workbench.aihub.qualcomm.com/jobs/j5687r80g/" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/audit_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/audit_report.json new file mode 100644 index 000000000..265ae861f --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/audit_report.json @@ -0,0 +1,79 @@ +{ + "scope": "one real Spark-X2.5 attention layer at one static 1024-token cache shape; no generation E2E", + "status": "component_executed_on_npu_quality_unqualified", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "compile_job_id": "jpyoxq885", + "inference_job_id": "jgzlm8yo5", + "profile_job_id": "j5wl78z3p", + "target_model_id": "mqkp10ezm", + "input_dataset_id": "d7m83gjy2", + "compile_options": "--target_runtime qnn_dlc --quantize_full_type int8 --quantize_io", + "files": { + "source_onnx": { + "bytes": 205618089, + "sha256": "4e2b1c8d791b1ee0e9b75f3322a20a743c0bf8c40f9b98edd963f2944d6300bb" + }, + "fixture": { + "bytes": 3901238, + "sha256": "5a439e6c6d148f1720938770daa110039a39edde014f1c1592c823c7d7dfc8ed" + }, + "source_cpu_output": { + "bytes": 12103, + "sha256": "6bbfb8e6ccb09475391ce1dbd2e855205720f4367c038a0d5be35e7dfeb2f768" + }, + "device_output": { + "bytes": 4865, + "sha256": "f927685b0398451268464f36bb0fa00e96507bd693302a3540e4aee0d7d8233c" + } + }, + "source_cpu_vs_device": { + "hidden": { + "relative_l2": 0.20913809283274212, + "max_abs": 0.035594239830970764, + "cosine": 0.9800697623054245 + }, + "k_new_0": { + "relative_l2": 0.01701873991186353, + "max_abs": 0.15612030029296875, + "cosine": 0.9998555562334065 + }, + "v_new_0": { + "relative_l2": 0.04413272608978388, + "max_abs": 0.07510209828615189, + "cosine": 0.9990311183867252 + } + }, + "profile": { + "sample_count": 100, + "sample_min_us": 5089, + "nearest_rank_p50_us": 5610, + "nearest_rank_p95_us": 5761, + "reported_estimated_inference_time_us": 5089, + "reported_peak_memory_bytes": 5951488, + "compute_unit_row_counts": { + "NPU": 61 + } + }, + "limits": [ + "No next-token or task-level numerical tolerance is established.", + "No complete prefill, KV/ring, sampling, continuous generation or Omni deployment is tested.", + "Hosted component latency is not a complete request or sustained thermal profile." + ] +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/device_output.npz b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/device_output.npz new file mode 100644 index 000000000..be1ee17e7 Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/device_output.npz differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/inference_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/inference_report.json new file mode 100644 index 000000000..50b5d8cf2 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/inference_report.json @@ -0,0 +1,32 @@ +{ + "timestamp_utc": "2026-09-23T22:06:32.836584+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jgzlm8yo5", + "url": "https://workbench.aihub.qualcomm.com/jobs/jgzlm8yo5/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "model_id": "mqkp10ezm", + "input_dataset_id": "d7m83gjy2", + "outputs": "device_output.npz" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/inference_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/inference_submission.json new file mode 100644 index 000000000..b0bceedc8 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/inference_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-23T21:56:26.436621+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jpyoxq885", + "target_model_id": "mqkp10ezm", + "input_dataset_id": "d7m83gjy2", + "device": "Dragonwing RB3 Gen 2 Vision Kit", + "options": "--compute_unit npu", + "kind": "inference", + "status": "SUBMITTED", + "job_id": "jgzlm8yo5", + "url": "https://workbench.aihub.qualcomm.com/jobs/jgzlm8yo5/" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/profile_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/profile_report.json new file mode 100644 index 000000000..ba8221f7b --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/profile_report.json @@ -0,0 +1,667 @@ +{ + "timestamp_utc": "2026-09-23T22:06:37.770571+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "j5wl78z3p", + "url": "https://workbench.aihub.qualcomm.com/jobs/j5wl78z3p/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Dragonwing RB3 Gen 2 Vision Kit", + "os": "1.6", + "attributes": [ + "os:qc_linux", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-oe-linux-gcc11", + "format:iot", + "chipset:qualcomm-qcs6490", + "chipset:qcs6490", + "chipset:qualcomm-dragonwing-qcs6490", + "hexagon:v68", + "soc-model:93", + "framework:qnn" + ] + }, + "model_id": "mqkp10ezm", + "shapes": { + "x": [ + [ + 1, + 1, + 2048 + ], + "uint8" + ], + "cos_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint8" + ], + "sin_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint8" + ], + "k_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint8" + ], + "v_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint8" + ], + "mask_full": [ + [ + 1, + 1, + 1, + 1025 + ], + "uint8" + ] + }, + "profile": { + "execution_summary": { + "estimated_inference_time": 5089, + "estimated_inference_peak_memory": 5951488, + "first_load_time": 1056655, + "first_load_peak_memory": 299769856, + "warm_load_time": 237131, + "warm_load_peak_memory": 8380416, + "compile_time": 0, + "compile_peak_memory": 0, + "compile_memory_increase_range": [ + 0, + 0 + ], + "compile_memory_peak_range": [ + 0, + 0 + ], + "first_load_memory_increase_range": [ + 87453872, + 88254496 + ], + "first_load_memory_peak_range": [ + 296873984, + 297674608 + ], + "warm_load_memory_increase_range": [ + 3804784, + 4606864 + ], + "warm_load_memory_peak_range": [ + 5480448, + 6282528 + ], + "inference_memory_increase_range": [ + 0, + 59568 + ], + "inference_memory_peak_range": [ + 565248, + 3277312 + ], + "all_compile_times": [], + "all_first_load_times": [ + 1056655 + ], + "all_warm_load_times": [ + 237131 + ], + "all_inference_times": [ + 6294, + 5914, + 5642, + 5540, + 5501, + 5839, + 5646, + 5125, + 5152, + 5089, + 5143, + 5610, + 5301, + 5596, + 5612, + 5710, + 5709, + 5444, + 5465, + 5572, + 5674, + 5678, + 5774, + 5458, + 5323, + 5424, + 5673, + 5682, + 5705, + 5601, + 5668, + 5701, + 5761, + 5711, + 5367, + 5426, + 5717, + 5544, + 5446, + 5522, + 5472, + 5453, + 5642, + 5754, + 5495, + 5653, + 5722, + 5362, + 5677, + 5475, + 5647, + 5680, + 5737, + 5440, + 5511, + 5725, + 5452, + 5465, + 5656, + 5436, + 5843, + 5700, + 5492, + 5421, + 5414, + 5689, + 5702, + 5277, + 5428, + 5434, + 5502, + 5648, + 5437, + 5727, + 5698, + 5440, + 5412, + 5705, + 5722, + 5704, + 5424, + 5439, + 5427, + 5353, + 5698, + 5432, + 5652, + 5692, + 5490, + 5400, + 5399, + 5635, + 5650, + 5708, + 5642, + 5704, + 5709, + 5694, + 5749, + 5443 + ] + }, + "execution_detail": [ + { + "name": "Input", + "type": "Input", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 356410 + }, + { + "name": "rms_norm_/layers.0/Mul_1_output_0", + "type": "rms", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 54755 + }, + { + "name": "/layers.0/q_k_v_proj/MatMul", + "type": "/layers.0/q", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/q_k_v_proj/MatMul_post_reshape", + "type": "/layers.0/q", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "node_Split_20_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 442481 + }, + { + "name": "node_Split_20_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 112903 + }, + { + "name": "node_Split_20_to_slice_2", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 121150 + }, + { + "name": "/layers.0/Reshape", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_1", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_2", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/g_proj/MatMul", + "type": "/layers.0/g", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 7832 + }, + { + "name": "/layers.0/g_proj/MatMul_post_reshape", + "type": "/layers.0/g", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Sigmoid", + "type": "/layers.0/Sigmoid", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1767 + }, + { + "name": "/layers.0/Reshape_3", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 677 + }, + { + "name": "node_Split_22_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 282 + }, + { + "name": "node_Split_22_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 323 + }, + { + "name": "fused_/layers.0/Slice_3_/layers.0/Slice_5", + "type": "fused", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 337 + }, + { + "name": "fused_/layers.0/Slice_3_/layers.0/Slice_6", + "type": "fused", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 451 + }, + { + "name": "/layers.0/Mul_2", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 602 + }, + { + "name": "/layers.0/Neg", + "type": "/layers.0/Neg", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 172 + }, + { + "name": "/layers.0/Concat", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 798 + }, + { + "name": "/layers.0/Mul_3", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1364 + }, + { + "name": "/layers.0/Add_1", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 961 + }, + { + "name": "/layers.0/Concat_1", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 660 + }, + { + "name": "node_Split_24_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 275 + }, + { + "name": "node_Split_24_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1825 + }, + { + "name": "fused_/layers.0/Slice_7_/layers.0/Slice_9", + "type": "fused", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 232 + }, + { + "name": "fused_/layers.0/Slice_7_/layers.0/Slice_10", + "type": "fused", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3753 + }, + { + "name": "/layers.0/Mul_4", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 581 + }, + { + "name": "/layers.0/Neg_1", + "type": "/layers.0/Neg", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 653 + }, + { + "name": "/layers.0/Concat_2", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1227 + }, + { + "name": "/layers.0/Mul_5", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1114 + }, + { + "name": "/layers.0/Add_2", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3012 + }, + { + "name": "/layers.0/Concat_3", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2091 + }, + { + "name": "/layers.0/Concat_4", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 9979 + }, + { + "name": "/layers.0/Concat_5", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_4", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2759 + }, + { + "name": "/layers.0/Transpose", + "type": "/layers.0/Transpose", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/MatMul", + "type": "/layers.0/MatMul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 414649 + }, + { + "name": "/layers.0/Mul_6", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3908 + }, + { + "name": "/layers.0/Add_3", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 6454 + }, + { + "name": "/layers.0/Softmax", + "type": "/layers.0/Softmax", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 10636 + }, + { + "name": "/layers.0/MatMul_1", + "type": "/layers.0/MatMul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 212095 + }, + { + "name": "/layers.0/Reshape_5", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_7", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 17068 + }, + { + "name": "/layers.0/out_proj/MatMul", + "type": "/layers.0/out", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 443675 + }, + { + "name": "/layers.0/out_proj/MatMul_post_reshape", + "type": "/layers.0/out", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_4", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 7210 + }, + { + "name": "rms_norm_/layers.0/Mul_9_output_0", + "type": "rms", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 24034 + }, + { + "name": "/layers.0/gate_proj/MatMul", + "type": "/layers.0/gate", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1434127 + }, + { + "name": "/layers.0/gate_proj/MatMul_post_reshape", + "type": "/layers.0/gate", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "node_Gelu_2", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 10469 + }, + { + "name": "/layers.0/up_proj/MatMul", + "type": "/layers.0/up", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1430626 + }, + { + "name": "/layers.0/up_proj/MatMul_post_reshape", + "type": "/layers.0/up", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_12", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 38676 + }, + { + "name": "/layers.0/down_proj/MatMul", + "type": "/layers.0/down", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1440368 + }, + { + "name": "/layers.0/down_proj/MatMul_post_reshape", + "type": "/layers.0/down", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_7", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1363 + }, + { + "name": "/layers.0/Concat_3_identity", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_2_identity", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "Output", + "type": "Output", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 10720 + } + ] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/profile_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/profile_submission.json new file mode 100644 index 000000000..59da5e480 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/w8a8/profile_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-23T21:56:28.293907+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jpyoxq885", + "target_model_id": "mqkp10ezm", + "input_dataset_id": null, + "device": "Dragonwing RB3 Gen 2 Vision Kit", + "options": "--compute_unit npu", + "kind": "profile", + "status": "SUBMITTED", + "job_id": "j5wl78z3p", + "url": "https://workbench.aihub.qualcomm.com/jobs/j5wl78z3p/" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/README.md new file mode 100644 index 000000000..5e9cf0a09 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/README.md @@ -0,0 +1,11 @@ +# Spark-X2.5 full-attention layer on Galaxy S24 NPU + +**One real Spark-X2.5-1.7B full-attention decoder layer ran on a hosted Samsung Galaxy S24 NPU; complete Spark generation did not.** This is a fixed decode-step component with a 1,024-token full-attention cache, not the 28-layer model, prefill, sampling, a resident KV/ring session or a device-local Omni controller. Qualcomm AI Hub Workbench provisions and releases a physical device for each component job; it is a test facility, not the deployment runtime. ADB still lists no attached device. + +The source is the existing FP32 `spark_full_ctx1024.onnx` export from real Spark weights. Its SHA256 is `4e2b1c8d791b1ee0e9b75f3322a20a743c0bf8c40f9b98edd963f2944d6300bb` (205,618,089 bytes); downloading uploaded source model `mn14dw6zn` yielded the same byte-for-byte hash. Workbench [quantization job](quantization_provenance.json) `jg9zne6wp` used calibration dataset `d7x8e3d59` and produced QDQ ONNX source model `mmd5e3v3q`. Its downloaded external-data ZIP SHA256 is `703821ca8ace5672228919ea2add4901def4887748c381b12ffc8b259665c553`; the graph has 92 QuantizeLinear/92 DequantizeLinear nodes, INT8 weight zero-points and UINT16 activation zero-points (W8A16). The original and quantized graphs and the exact inference fixture are stored outside Git under `/home/zhout/project/edge_infer/models/`; the [audit](audit_report.json) pins their sizes and hashes. + +The [compile job](compile_report.json) succeeded for exact device **Samsung Galaxy S24, Android 14**, using Workbench client `0.55.0`, service build recorded in the job JSON, target model `mq9y70k0n` and `--target_runtime qnn_dlc --quantize_io`. The old context-binary target was [rejected at submission](compile_legacy_option_rejection.json) by the current API; [Qualcomm's compile options](https://workbench.aihub.qualcomm.com/docs/hub/api.html) list `qnn_dlc` as a current target. The [inference job](inference_report.json) used the pinned dataset `d7m83gjy2` and requested `--compute_unit npu`; its three finite FP32 outputs are [retained](s24_output.npz). The [profile job](profile_report.json) also requested NPU and attributed all 73 execution-detail rows to NPU. Its 100 `all_inference_times` samples had nearest-rank p50/p95 **1,109/1,205 microseconds** (minimum 1,098 microseconds); the separate `estimated_inference_time` field equals the minimum and is not a median. Reported inference peak memory was 210,259,968 bytes. These are component-job figures; they exclude controller, transfers between stages, prefill, all other layers, sampling, loading and sustained heat. + +The fixed fixture has FP32 input `[1,1,2048]`, cache tensors `[1,2,1024,256]`, rotary angles and mask. Local ONNX Runtime 1.29 CPU executions of the original FP32 source and downloaded QDQ source produced separate pinned outputs. Relative L2 errors for `hidden` / new key / new value were **1.236% / 0.376% / 0.651%** for QDQ CPU versus FP32 CPU, then **0.555% / 0.0065% / 0.0126%** for S24 NPU versus QDQ CPU. S24 and the prior S25 [NPU output](s25_output_prior.npz) matched exactly on the same dataset ([S25 inference provenance](s25_inference_prior.json), [S25 profile](s25_profile_prior.json)), although they used different compiled target formats and Android versions. These measurements suggest quantization contributes substantially to the source-to-device difference; they do **not** establish next-token or task-quality tolerance. The [audit report](audit_report.json) retains full precision metrics, cosines, artifact hashes, sample statistics and job IDs. The [audit script](../../../../experiments/audit_spark_qualcomm_attention.py) reproduces those comparisons with the downloaded artifacts and fixture. + +The matrix remains **NOT E2E**. The next gate is a device-resident, complete Spark prefill/continuous-decode/KV-ring/sampling loop with explicit state lifecycle, numeric token-quality checks, 128-token generation across 512/1024 context transitions, cancellation, memory admission and sustained thermal/power profiling. Workbench component jobs cannot substitute for that local runtime. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/audit_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/audit_report.json new file mode 100644 index 000000000..a151b8626 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/audit_report.json @@ -0,0 +1,143 @@ +{ + "scope": "one real Spark-X2.5 full-attention decoder layer at one static 1024-token cache shape; no generation E2E", + "status": "component_executed_on_npu_quality_unqualified", + "device": { + "name": "Samsung Galaxy S24", + "os": "14", + "attributes": [ + "os:android", + "framework:tflite", + "framework:onnx", + "abi:aarch64-android", + "vendor:samsung", + "format:phone", + "chipset:qualcomm-snapdragon-8gen3", + "chipset:sm8650", + "hexagon:v75", + "soc-model:57", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "compile_job_id": "jpe7z8l15", + "inference_job_id": "j57e4k9lp", + "profile_job_id": "jp4y1m3vp", + "target_model_id": "mq9y70k0n", + "input_dataset_id": "d7m83gjy2", + "s25_prior_inference_job_id": "jp0mvkj6g", + "files": { + "source_onnx": { + "bytes": 205618089, + "sha256": "4e2b1c8d791b1ee0e9b75f3322a20a743c0bf8c40f9b98edd963f2944d6300bb" + }, + "quantized_onnx_zip": { + "bytes": 186772390, + "sha256": "703821ca8ace5672228919ea2add4901def4887748c381b12ffc8b259665c553" + }, + "fixture": { + "bytes": 3901238, + "sha256": "5a439e6c6d148f1720938770daa110039a39edde014f1c1592c823c7d7dfc8ed" + }, + "source_cpu_output": { + "bytes": 12103, + "sha256": "6bbfb8e6ccb09475391ce1dbd2e855205720f4367c038a0d5be35e7dfeb2f768" + }, + "quantized_cpu_output": { + "bytes": 11574, + "sha256": "061654773166d102a11228c4a2e4cd06de6c2482c7e4e7860315b7b9a95fa8a1" + }, + "s25_output": { + "bytes": 13072, + "sha256": "2dea9f4b478cb02b9be74a8d3ea4ad42e562e4de72413967d22706a1b6c6b9c0" + }, + "device_output": { + "bytes": 11576, + "sha256": "9381fccee14b303edc5b432edf92422f2948e111ea6848df22cba3cd289a7073" + } + }, + "source_vs_quantized_cpu": { + "hidden": { + "relative_l2": 0.012364819167814612, + "max_abs": 0.0028282999992370605, + "cosine": 0.9999236499353131 + }, + "k_new_0": { + "relative_l2": 0.0037627070087598766, + "max_abs": 0.03641480207443237, + "cosine": 0.9999929226489408 + }, + "v_new_0": { + "relative_l2": 0.006507432119585315, + "max_abs": 0.011739224195480347, + "cosine": 0.9999789079393114 + } + }, + "quantized_cpu_vs_device_npu": { + "hidden": { + "relative_l2": 0.005550082388164841, + "max_abs": 0.002016603946685791, + "cosine": 0.9999847274053486 + }, + "k_new_0": { + "relative_l2": 6.533750243101402e-05, + "max_abs": 0.00033855438232421875, + "cosine": 0.9999999978819449 + }, + "v_new_0": { + "relative_l2": 0.0001257028618212242, + "max_abs": 0.0002911090850830078, + "cosine": 0.9999999921460195 + } + }, + "source_cpu_vs_device_npu": { + "hidden": { + "relative_l2": 0.01350919207884596, + "max_abs": 0.0048449039459228516, + "cosine": 0.9999091967737982 + }, + "k_new_0": { + "relative_l2": 0.003766040678719007, + "max_abs": 0.03641480207443237, + "cosine": 0.9999929097855774 + }, + "v_new_0": { + "relative_l2": 0.006504874746412689, + "max_abs": 0.011739224195480347, + "cosine": 0.9999789285134126 + } + }, + "s25_npu_vs_device_npu": { + "hidden": { + "relative_l2": 0.0, + "max_abs": 0.0, + "cosine": 0.9999999999999998 + }, + "k_new_0": { + "relative_l2": 0.0, + "max_abs": 0.0, + "cosine": 0.9999999999999998 + }, + "v_new_0": { + "relative_l2": 0.0, + "max_abs": 0.0, + "cosine": 1.0 + } + }, + "profile": { + "sample_count": 100, + "sample_min_us": 1098, + "nearest_rank_p50_us": 1109, + "nearest_rank_p95_us": 1205, + "reported_estimated_inference_time_us": 1098, + "reported_peak_memory_bytes": 210259968, + "compute_unit_row_counts": { + "NPU": 73 + } + }, + "limits": [ + "No task-level numerical tolerance or next-token quality gate is established.", + "Hub component jobs do not prove device-local prefill, KV/ring update, sampling, continuous generation or Omni deployment.", + "Profile sample latency is one component, not complete-request or sustained thermal performance." + ] +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/compile_legacy_option_rejection.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/compile_legacy_option_rejection.json new file mode 100644 index 000000000..d199ebb31 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/compile_legacy_option_rejection.json @@ -0,0 +1,9 @@ +{ + "timestamp_utc": "2026-09-23T21:12:36.038442+00:00", + "qai_hub_version": "0.55.0", + "device": "Samsung Galaxy S24", + "source_quantized_model_id": "mmd5e3v3q", + "requested_options": "--target_runtime qnn_context_binary --quantize_io", + "status": "REJECTED_AT_SUBMISSION", + "message": "Unsupported target runtime: qnn_context_binary. Supported values reported by API: onnx, precompiled_qnn_onnx, qnn_dlc, tflite." +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/compile_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/compile_report.json new file mode 100644 index 000000000..de457f3cd --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/compile_report.json @@ -0,0 +1,32 @@ +{ + "timestamp_utc": "2026-09-23T21:15:44.477548+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jpe7z8l15", + "url": "https://workbench.aihub.qualcomm.com/jobs/jpe7z8l15/", + "status": "SUCCESS", + "status_message": "", + "source_quantized_model_id": "mmd5e3v3q", + "target_model_id": "mq9y70k0n", + "device": { + "name": "Samsung Galaxy S24", + "os": "14", + "attributes": [ + "os:android", + "framework:tflite", + "framework:onnx", + "abi:aarch64-android", + "vendor:samsung", + "format:phone", + "chipset:qualcomm-snapdragon-8gen3", + "chipset:sm8650", + "hexagon:v75", + "soc-model:57", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "options": "--target_runtime qnn_dlc --quantize_io", + "hub_version": "aihub-2026.09.11.0" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/compile_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/compile_submission.json new file mode 100644 index 000000000..9800dead2 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/compile_submission.json @@ -0,0 +1,11 @@ +{ + "timestamp_utc": "2026-09-23T21:12:39.099981+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "job_id": "jpe7z8l15", + "url": "https://workbench.aihub.qualcomm.com/jobs/jpe7z8l15/", + "source_quantized_model_id": "mmd5e3v3q", + "source_onnx_sha256": "4e2b1c8d791b1ee0e9b75f3322a20a743c0bf8c40f9b98edd963f2944d6300bb", + "device": "Samsung Galaxy S24", + "options": "--target_runtime qnn_dlc --quantize_io" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/inference_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/inference_report.json new file mode 100644 index 000000000..d2f447e89 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/inference_report.json @@ -0,0 +1,33 @@ +{ + "timestamp_utc": "2026-09-23T21:19:44.223245+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "j57e4k9lp", + "url": "https://workbench.aihub.qualcomm.com/jobs/j57e4k9lp/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Samsung Galaxy S24", + "os": "14", + "attributes": [ + "os:android", + "framework:tflite", + "framework:onnx", + "abi:aarch64-android", + "vendor:samsung", + "format:phone", + "chipset:qualcomm-snapdragon-8gen3", + "chipset:sm8650", + "hexagon:v75", + "soc-model:57", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "model_id": "mq9y70k0n", + "input_dataset_id": "d7m83gjy2", + "outputs": "s24_output.npz" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/inference_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/inference_submission.json new file mode 100644 index 000000000..f7a41c810 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/inference_submission.json @@ -0,0 +1,9 @@ +{ + "timestamp_utc": "2026-09-23T21:15:46.475323+00:00", + "job_id": "j57e4k9lp", + "url": "https://workbench.aihub.qualcomm.com/jobs/j57e4k9lp/", + "target_model_id": "mq9y70k0n", + "device": "Samsung Galaxy S24", + "input_dataset_id": "d7m83gjy2", + "options": "--compute_unit npu" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/profile_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/profile_report.json new file mode 100644 index 000000000..4dcbc6445 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/profile_report.json @@ -0,0 +1,752 @@ +{ + "timestamp_utc": "2026-09-23T21:19:48.832365+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jp4y1m3vp", + "url": "https://workbench.aihub.qualcomm.com/jobs/jp4y1m3vp/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Samsung Galaxy S24", + "os": "14", + "attributes": [ + "os:android", + "framework:tflite", + "framework:onnx", + "abi:aarch64-android", + "vendor:samsung", + "format:phone", + "chipset:qualcomm-snapdragon-8gen3", + "chipset:sm8650", + "hexagon:v75", + "soc-model:57", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "model_id": "mq9y70k0n", + "shapes": { + "x": [ + [ + 1, + 1, + 2048 + ], + "uint16" + ], + "v_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint16" + ], + "cos_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint16" + ], + "sin_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint16" + ], + "k_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint16" + ], + "mask_full": [ + [ + 1, + 1, + 1, + 1025 + ], + "uint16" + ] + }, + "profile": { + "execution_summary": { + "estimated_inference_time": 1098, + "estimated_inference_peak_memory": 210259968, + "first_load_time": 943277, + "first_load_peak_memory": 340230144, + "warm_load_time": 174148, + "warm_load_peak_memory": 214511616, + "compile_time": 0, + "compile_peak_memory": 0, + "compile_memory_increase_range": [ + 0, + 0 + ], + "compile_memory_peak_range": [ + 0, + 0 + ], + "first_load_memory_increase_range": [ + 62276784, + 69247552 + ], + "first_load_memory_peak_range": [ + 258228224, + 265198992 + ], + "warm_load_memory_increase_range": [ + 0, + 1646624 + ], + "warm_load_memory_peak_range": [ + 606208, + 113631744 + ], + "inference_memory_increase_range": [ + 0, + 0 + ], + "inference_memory_peak_range": [ + 20480, + 111512672 + ], + "all_compile_times": [], + "all_first_load_times": [ + 943277 + ], + "all_warm_load_times": [ + 174148 + ], + "all_inference_times": [ + 1834, + 1201, + 1153, + 1156, + 1165, + 1143, + 1133, + 1141, + 1170, + 1150, + 1144, + 1425, + 1225, + 1205, + 1194, + 1212, + 1106, + 1103, + 1104, + 1098, + 1098, + 1098, + 1107, + 1100, + 1110, + 1102, + 1103, + 1100, + 1099, + 1109, + 1151, + 1120, + 1116, + 1131, + 1108, + 1107, + 1107, + 1118, + 1292, + 1123, + 1109, + 1115, + 1111, + 1107, + 1107, + 1111, + 1111, + 1107, + 1112, + 1105, + 1110, + 1107, + 1106, + 1121, + 1106, + 1107, + 1106, + 1105, + 1105, + 1123, + 1118, + 1111, + 1120, + 1109, + 1105, + 1104, + 1102, + 1115, + 1105, + 1108, + 1108, + 1110, + 1104, + 1108, + 1109, + 1139, + 1130, + 1125, + 1124, + 1118, + 1104, + 1098, + 1119, + 1098, + 1099, + 1098, + 1107, + 1102, + 1135, + 1117, + 1131, + 1106, + 1103, + 1121, + 1109, + 1109, + 1117, + 1111, + 1106, + 1107 + ] + }, + "execution_detail": [ + { + "name": "Input", + "type": "Input", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 42949 + }, + { + "name": "/layers.0/Pow", + "type": "/layers.0/Pow", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 41382 + }, + { + "name": "/layers.0/ReduceMean", + "type": "/layers.0/ReduceMean", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 8204 + }, + { + "name": "/layers.0/Add", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 15143 + }, + { + "name": "/layers.0/Sqrt", + "type": "/layers.0/Sqrt", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Div", + "type": "/layers.0/Div", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 82056 + }, + { + "name": "/layers.0/Mul", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 61350 + }, + { + "name": "/layers.0/Mul_1", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 17794 + }, + { + "name": "/layers.0/q_k_v_proj/MatMul", + "type": "/layers.0/q", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/q_k_v_proj/MatMul_post_reshape", + "type": "/layers.0/q", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/g_proj/MatMul", + "type": "/layers.0/g", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 6373 + }, + { + "name": "/layers.0/g_proj/MatMul_post_reshape", + "type": "/layers.0/g", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "node_Split_1_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 44219 + }, + { + "name": "node_Split_1_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 23821 + }, + { + "name": "node_Split_1_to_slice_2", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 9323 + }, + { + "name": "/layers.0/Sigmoid", + "type": "/layers.0/Sigmoid", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_1", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_2", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 6592 + }, + { + "name": "/layers.0/Reshape_3", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 8691 + }, + { + "name": "node_Split_3_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 456 + }, + { + "name": "node_Split_3_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1295 + }, + { + "name": "node_Split_5_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 340 + }, + { + "name": "node_Split_5_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5592 + }, + { + "name": "/layers.0/Concat_5", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 31506 + }, + { + "name": "node_Split_7_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1508 + }, + { + "name": "node_Split_7_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 471 + }, + { + "name": "/layers.0/Mul_2", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2954 + }, + { + "name": "node_Split_9_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 4193 + }, + { + "name": "node_Split_9_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1297 + }, + { + "name": "/layers.0/Mul_4", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 6658 + }, + { + "name": "/layers.0/Neg", + "type": "/layers.0/Neg", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1367 + }, + { + "name": "/layers.0/Neg_1", + "type": "/layers.0/Neg", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 4097 + }, + { + "name": "/layers.0/Concat", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 727 + }, + { + "name": "/layers.0/Concat_2", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3507 + }, + { + "name": "/layers.0/Mul_3", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3769 + }, + { + "name": "/layers.0/Mul_5", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2749 + }, + { + "name": "/layers.0/Add_1", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1524 + }, + { + "name": "/layers.0/Add_2", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 4679 + }, + { + "name": "/layers.0/Concat_1", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 833 + }, + { + "name": "/layers.0/Concat_3", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 791 + }, + { + "name": "/layers.0/Reshape_4", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 248 + }, + { + "name": "/layers.0/Concat_4", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Transpose", + "type": "/layers.0/Transpose", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/MatMul", + "type": "/layers.0/MatMul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 685248 + }, + { + "name": "/layers.0/Mul_6", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2929 + }, + { + "name": "/layers.0/Add_3", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 26814 + }, + { + "name": "/layers.0/Softmax", + "type": "/layers.0/Softmax", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 17675 + }, + { + "name": "/layers.0/MatMul_1", + "type": "/layers.0/MatMul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 637945 + }, + { + "name": "/layers.0/Reshape_5", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_7", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 22264 + }, + { + "name": "/layers.0/out_proj/MatMul", + "type": "/layers.0/out", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 66743 + }, + { + "name": "/layers.0/out_proj/MatMul_post_reshape", + "type": "/layers.0/out", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_4", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 6574 + }, + { + "name": "/layers.0/Pow_1", + "type": "/layers.0/Pow", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3443 + }, + { + "name": "/layers.0/ReduceMean_1", + "type": "/layers.0/ReduceMean", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1522 + }, + { + "name": "/layers.0/Add_5", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 179 + }, + { + "name": "/layers.0/Sqrt_1", + "type": "/layers.0/Sqrt", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Div_1", + "type": "/layers.0/Div", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 57936 + }, + { + "name": "/layers.0/Mul_8", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 34269 + }, + { + "name": "/layers.0/Mul_9", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 7526 + }, + { + "name": "/layers.0/gate_proj/MatMul", + "type": "/layers.0/gate", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/gate_proj/MatMul_post_reshape", + "type": "/layers.0/gate", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/up_proj/MatMul", + "type": "/layers.0/up", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 426843 + }, + { + "name": "/layers.0/up_proj/MatMul_post_reshape", + "type": "/layers.0/up", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "_elementwiseneuron_0", + "type": "elementwiseneuron", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 419979 + }, + { + "name": "/layers.0/Mul_12", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 12346 + }, + { + "name": "/layers.0/down_proj/MatMul", + "type": "/layers.0/down", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 406308 + }, + { + "name": "/layers.0/down_proj/MatMul_post_reshape", + "type": "/layers.0/down", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_7", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5803 + }, + { + "name": "/layers.0/Concat_3_identity", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_2_identity", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "Output", + "type": "Output", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 8424 + } + ] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/profile_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/profile_submission.json new file mode 100644 index 000000000..d5354e465 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/profile_submission.json @@ -0,0 +1,9 @@ +{ + "timestamp_utc": "2026-09-23T21:15:47.628043+00:00", + "job_id": "jp4y1m3vp", + "url": "https://workbench.aihub.qualcomm.com/jobs/jp4y1m3vp/", + "target_model_id": "mq9y70k0n", + "device": "Samsung Galaxy S24", + "input_dataset_id": null, + "options": "--compute_unit npu" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/quantization_provenance.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/quantization_provenance.json new file mode 100644 index 000000000..bba7e061a --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/quantization_provenance.json @@ -0,0 +1,17 @@ +{ + "timestamp_utc": "2026-09-23T21:25:02.276846+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jg9zne6wp", + "url": "https://workbench.aihub.qualcomm.com/jobs/jg9zne6wp/", + "status": "SUCCESS", + "status_message": "", + "source_model_id": "mn14dw6zn", + "source_model_name": "spark25_full1024_src", + "source_download_sha256": "4e2b1c8d791b1ee0e9b75f3322a20a743c0bf8c40f9b98edd963f2944d6300bb", + "calibration_dataset_id": "d7x8e3d59", + "target_quantized_model_id": "mmd5e3v3q", + "downloaded_qdq_zip_sha256": "703821ca8ace5672228919ea2add4901def4887748c381b12ffc8b259665c553", + "options": "", + "hub_version": "aihub-2026.08.28.1" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/s24_output.npz b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/s24_output.npz new file mode 100644 index 000000000..f00cd218e Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/s24_output.npz differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/s25_inference_prior.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/s25_inference_prior.json new file mode 100644 index 000000000..4c564dfcc --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/s25_inference_prior.json @@ -0,0 +1,34 @@ +{ + "timestamp_utc": "2026-09-22T03:52:48.169689+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "command": "collect", + "job_id": "jp0mvkj6g", + "url": "https://workbench.aihub.qualcomm.com/jobs/jp0mvkj6g/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Samsung Galaxy S25", + "os": "15", + "attributes": [ + "os:android", + "framework:tflite", + "framework:onnx", + "abi:aarch64-android", + "vendor:samsung", + "format:phone", + "chipset:qualcomm-snapdragon-8-elite-for-galaxy", + "chipset:sm8750-ac", + "hexagon:v79", + "soc-model:69", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "model_id": "mnowl6xpq", + "input_dataset_id": "d7m83gjy2", + "outputs": "aihub-s25-infer.npz" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/s25_output_prior.npz b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/s25_output_prior.npz new file mode 100644 index 000000000..4248d167e Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/s25_output_prior.npz differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/s25_profile_prior.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/s25_profile_prior.json new file mode 100644 index 000000000..1fadce6e1 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/s25_profile_prior.json @@ -0,0 +1,753 @@ +{ + "timestamp_utc": "2026-09-22T03:53:16.578078+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "command": "collect", + "job_id": "jpyo7nx75", + "url": "https://workbench.aihub.qualcomm.com/jobs/jpyo7nx75/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Samsung Galaxy S25", + "os": "15", + "attributes": [ + "os:android", + "framework:tflite", + "framework:onnx", + "abi:aarch64-android", + "vendor:samsung", + "format:phone", + "chipset:qualcomm-snapdragon-8-elite-for-galaxy", + "chipset:sm8750-ac", + "hexagon:v79", + "soc-model:69", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "model_id": "mnowl6xpq", + "shapes": { + "x": [ + [ + 1, + 1, + 2048 + ], + "uint16" + ], + "v_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint16" + ], + "cos_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint16" + ], + "sin_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint16" + ], + "k_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint16" + ], + "mask_full": [ + [ + 1, + 1, + 1, + 1025 + ], + "uint16" + ] + }, + "profile": { + "execution_summary": { + "estimated_inference_time": 945, + "estimated_inference_peak_memory": 140197888, + "first_load_time": 113160, + "first_load_peak_memory": 113930240, + "warm_load_time": 113545, + "warm_load_peak_memory": 125128704, + "compile_time": 0, + "compile_peak_memory": 0, + "compile_memory_increase_range": [ + 0, + 0 + ], + "compile_memory_peak_range": [ + 0, + 0 + ], + "first_load_memory_increase_range": [ + 0, + 2327312 + ], + "first_load_memory_peak_range": [ + 1159168, + 10519888 + ], + "warm_load_memory_increase_range": [ + 0, + 1550192 + ], + "warm_load_memory_peak_range": [ + 0, + 9639312 + ], + "inference_memory_increase_range": [ + 0, + 2138160 + ], + "inference_memory_peak_range": [ + 0, + 9296720 + ], + "all_compile_times": [], + "all_first_load_times": [ + 113160 + ], + "all_warm_load_times": [ + 113545 + ], + "all_inference_times": [ + 1683, + 1022, + 967, + 956, + 957, + 947, + 973, + 951, + 950, + 957, + 953, + 1142, + 996, + 980, + 977, + 1006, + 973, + 976, + 982, + 1005, + 978, + 973, + 975, + 980, + 963, + 997, + 958, + 973, + 994, + 1028, + 1015, + 1004, + 1030, + 1017, + 1052, + 1092, + 971, + 953, + 950, + 951, + 947, + 958, + 947, + 945, + 970, + 948, + 950, + 947, + 954, + 949, + 957, + 970, + 953, + 949, + 950, + 952, + 949, + 950, + 947, + 952, + 957, + 949, + 948, + 952, + 948, + 957, + 945, + 952, + 961, + 946, + 947, + 948, + 947, + 954, + 947, + 948, + 947, + 970, + 951, + 946, + 990, + 955, + 987, + 959, + 952, + 997, + 967, + 948, + 947, + 945, + 950, + 959, + 947, + 945, + 950, + 991, + 990, + 985, + 978, + 962 + ] + }, + "execution_detail": [ + { + "name": "Input", + "type": "Input", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 27823 + }, + { + "name": "/layers.0/Pow", + "type": "/layers.0/Pow", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 39594 + }, + { + "name": "/layers.0/ReduceMean", + "type": "/layers.0/ReduceMean", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 9352 + }, + { + "name": "/layers.0/Add", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5344 + }, + { + "name": "/layers.0/Sqrt", + "type": "/layers.0/Sqrt", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Div", + "type": "/layers.0/Div", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 83375 + }, + { + "name": "/layers.0/Mul", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 72302 + }, + { + "name": "/layers.0/Mul_1", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 17867 + }, + { + "name": "/layers.0/q_k_v_proj/MatMul", + "type": "/layers.0/q", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/q_k_v_proj/MatMul_post_reshape", + "type": "/layers.0/q", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/g_proj/MatMul", + "type": "/layers.0/g", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 9085 + }, + { + "name": "/layers.0/g_proj/MatMul_post_reshape", + "type": "/layers.0/g", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "node_Slice_748", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 50418 + }, + { + "name": "node_Slice_751", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 20704 + }, + { + "name": "node_Slice_754", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 6418 + }, + { + "name": "/layers.0/Sigmoid", + "type": "/layers.0/Sigmoid", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_1", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_2", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5771 + }, + { + "name": "/layers.0/Reshape_3", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 18115 + }, + { + "name": "node_Slice_531", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 506 + }, + { + "name": "node_Slice_534", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1788 + }, + { + "name": "node_Slice_538", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 760 + }, + { + "name": "node_Slice_541", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 9602 + }, + { + "name": "/layers.0/Concat_5", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 26383 + }, + { + "name": "node_Slice_545", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1228 + }, + { + "name": "node_Slice_548", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 573 + }, + { + "name": "/layers.0/Mul_2", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3456 + }, + { + "name": "node_Slice_1174", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 8335 + }, + { + "name": "node_Slice_1177", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 866 + }, + { + "name": "/layers.0/Mul_4", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 19251 + }, + { + "name": "/layers.0/Neg", + "type": "/layers.0/Neg", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 788 + }, + { + "name": "/layers.0/Neg_1", + "type": "/layers.0/Neg", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5970 + }, + { + "name": "/layers.0/Concat", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 886 + }, + { + "name": "/layers.0/Concat_2", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3254 + }, + { + "name": "/layers.0/Mul_3", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3302 + }, + { + "name": "/layers.0/Mul_5", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2889 + }, + { + "name": "/layers.0/Add_1", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1082 + }, + { + "name": "/layers.0/Add_2", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 4002 + }, + { + "name": "/layers.0/Concat_1", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 823 + }, + { + "name": "/layers.0/Concat_3", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 842 + }, + { + "name": "/layers.0/Reshape_4", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 319 + }, + { + "name": "/layers.0/Concat_4", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Transpose", + "type": "/layers.0/Transpose", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/MatMul", + "type": "/layers.0/MatMul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 867826 + }, + { + "name": "/layers.0/Mul_6", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2467 + }, + { + "name": "/layers.0/Add_3", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 50892 + }, + { + "name": "/layers.0/Softmax", + "type": "/layers.0/Softmax", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 13765 + }, + { + "name": "/layers.0/MatMul_1", + "type": "/layers.0/MatMul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 611657 + }, + { + "name": "/layers.0/Reshape_5", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_7", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 29593 + }, + { + "name": "/layers.0/out_proj/MatMul", + "type": "/layers.0/out", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 53626 + }, + { + "name": "/layers.0/out_proj/MatMul_post_reshape", + "type": "/layers.0/out", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_4", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 7114 + }, + { + "name": "/layers.0/Pow_1", + "type": "/layers.0/Pow", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 8995 + }, + { + "name": "/layers.0/ReduceMean_1", + "type": "/layers.0/ReduceMean", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1481 + }, + { + "name": "/layers.0/Add_5", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 196 + }, + { + "name": "/layers.0/Sqrt_1", + "type": "/layers.0/Sqrt", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Div_1", + "type": "/layers.0/Div", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 60120 + }, + { + "name": "/layers.0/Mul_8", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 42681 + }, + { + "name": "/layers.0/Mul_9", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 8458 + }, + { + "name": "/layers.0/gate_proj/MatMul", + "type": "/layers.0/gate", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/gate_proj/MatMul_post_reshape", + "type": "/layers.0/gate", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/up_proj/MatMul", + "type": "/layers.0/up", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 389054 + }, + { + "name": "/layers.0/up_proj/MatMul_post_reshape", + "type": "/layers.0/up", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "_elementwiseneuron_0", + "type": "elementwiseneuron", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 395384 + }, + { + "name": "/layers.0/Mul_12", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 18593 + }, + { + "name": "/layers.0/down_proj/MatMul", + "type": "/layers.0/down", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 415523 + }, + { + "name": "/layers.0/down_proj/MatMul_post_reshape", + "type": "/layers.0/down", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_7", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5800 + }, + { + "name": "/layers.0/Concat_3_identity", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_2_identity", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "Output", + "type": "Output", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 7431 + } + ] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/README.md new file mode 100644 index 000000000..37cb5a345 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/README.md @@ -0,0 +1,13 @@ +# Spark-X2.5 attention layer on SA8775P ADP + +**One fixed-shape attention layer from the real Spark-X2.5-1.7B checkpoint executed on the hosted SA8775P ADP NPU (Android 14). This is component evidence, not complete Spark generation or an embedded Omni deployment.** + +The [Galaxy S24 provenance record](../spark_s24_full_attention/README.md) pins the FP32 source ONNX, W8A16 QDQ ONNX, calibration and exact input fixture. Its QNN DLC compile job `jpe7z8l15` produced target model `mq9y70k0n`; the same DLC and fixture dataset `d7m83gjy2` were reused here without recompilation. [Inference submission](inference_submission.json) job `jp1nj7j2g` and [profile submission](profile_submission.json) job `jgdd383eg` both requested `--compute_unit npu` for exact device **SA8775P ADP**. The completed [inference](inference_report.json) and [profile](profile_report.json) reports pin service build, Android version and chipset. + +The [three outputs](device_output.npz) were finite FP32 hidden/new-key/new-value tensors at a 1,024-token cache shape. They were bitwise identical to the earlier S24/S25 NPU outputs on the same fixture. The [audit](audit_report.json) measured hidden/key/value relative L2 of **1.351%/0.377%/0.650%** against original FP32 ONNX CPU and **0.555%/0.0065%/0.0126%** against QDQ ONNX CPU. No next-token or task-quality tolerance has been established. + +All 73 execution-detail rows in the profile were attributed to NPU. The 100 component samples had nearest-rank p50/p95 **1.837/2.079 ms** (minimum 1.749 ms); the service's `estimated_inference_time` was that minimum, not the median. Reported component inference peak memory was 90,877,952 bytes. These hosted component measurements exclude prefill, 27 other decoder layers, sampling, handoff, loading and a resident KV/ring/controller. They are not paired speed comparisons with other devices or OS builds. + +The [audit script](../../../../experiments/audit_spark_qualcomm_attention.py) reproduces the provenance and numerical checks using pinned source artifact paths in the S24 record. The [collector](../../../../experiments/collect_spark_aihub_component.py) retrieves these existing jobs via the local client configuration; neither script contains credentials. + +**Matrix status: component NPU inference/profiled; NOT E2E.** Full qualification needs a complete device-resident Spark artifact and Omni backend with prefill/decode/KV-ring/sampling, state, token quality, cancellation and memory admission, followed by sustained power and thermal measurement. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/audit_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/audit_report.json new file mode 100644 index 000000000..4546ea638 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/audit_report.json @@ -0,0 +1,142 @@ +{ + "scope": "one real Spark-X2.5 full-attention decoder layer at one static 1024-token cache shape; no generation E2E", + "status": "component_executed_on_npu_quality_unqualified", + "device": { + "name": "SA8775P ADP", + "os": "14", + "attributes": [ + "os:android", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-android", + "format:auto", + "chipset:qualcomm-sa8775p", + "hexagon:v73", + "soc-model:52", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "compile_job_id": "jpe7z8l15", + "inference_job_id": "jp1nj7j2g", + "profile_job_id": "jgdd383eg", + "target_model_id": "mq9y70k0n", + "input_dataset_id": "d7m83gjy2", + "s25_prior_inference_job_id": "jp0mvkj6g", + "files": { + "source_onnx": { + "bytes": 205618089, + "sha256": "4e2b1c8d791b1ee0e9b75f3322a20a743c0bf8c40f9b98edd963f2944d6300bb" + }, + "quantized_onnx_zip": { + "bytes": 186772390, + "sha256": "703821ca8ace5672228919ea2add4901def4887748c381b12ffc8b259665c553" + }, + "fixture": { + "bytes": 3901238, + "sha256": "5a439e6c6d148f1720938770daa110039a39edde014f1c1592c823c7d7dfc8ed" + }, + "source_cpu_output": { + "bytes": 12103, + "sha256": "6bbfb8e6ccb09475391ce1dbd2e855205720f4367c038a0d5be35e7dfeb2f768" + }, + "quantized_cpu_output": { + "bytes": 11574, + "sha256": "061654773166d102a11228c4a2e4cd06de6c2482c7e4e7860315b7b9a95fa8a1" + }, + "s25_output": { + "bytes": 13072, + "sha256": "2dea9f4b478cb02b9be74a8d3ea4ad42e562e4de72413967d22706a1b6c6b9c0" + }, + "device_output": { + "bytes": 11576, + "sha256": "9381fccee14b303edc5b432edf92422f2948e111ea6848df22cba3cd289a7073" + } + }, + "source_vs_quantized_cpu": { + "hidden": { + "relative_l2": 0.012364819167814612, + "max_abs": 0.0028282999992370605, + "cosine": 0.9999236499353131 + }, + "k_new_0": { + "relative_l2": 0.0037627070087598766, + "max_abs": 0.03641480207443237, + "cosine": 0.9999929226489408 + }, + "v_new_0": { + "relative_l2": 0.006507432119585315, + "max_abs": 0.011739224195480347, + "cosine": 0.9999789079393114 + } + }, + "quantized_cpu_vs_device_npu": { + "hidden": { + "relative_l2": 0.005550082388164841, + "max_abs": 0.002016603946685791, + "cosine": 0.9999847274053486 + }, + "k_new_0": { + "relative_l2": 6.533750243101402e-05, + "max_abs": 0.00033855438232421875, + "cosine": 0.9999999978819449 + }, + "v_new_0": { + "relative_l2": 0.0001257028618212242, + "max_abs": 0.0002911090850830078, + "cosine": 0.9999999921460195 + } + }, + "source_cpu_vs_device_npu": { + "hidden": { + "relative_l2": 0.01350919207884596, + "max_abs": 0.0048449039459228516, + "cosine": 0.9999091967737982 + }, + "k_new_0": { + "relative_l2": 0.003766040678719007, + "max_abs": 0.03641480207443237, + "cosine": 0.9999929097855774 + }, + "v_new_0": { + "relative_l2": 0.006504874746412689, + "max_abs": 0.011739224195480347, + "cosine": 0.9999789285134126 + } + }, + "s25_npu_vs_device_npu": { + "hidden": { + "relative_l2": 0.0, + "max_abs": 0.0, + "cosine": 0.9999999999999998 + }, + "k_new_0": { + "relative_l2": 0.0, + "max_abs": 0.0, + "cosine": 0.9999999999999998 + }, + "v_new_0": { + "relative_l2": 0.0, + "max_abs": 0.0, + "cosine": 1.0 + } + }, + "profile": { + "sample_count": 100, + "sample_min_us": 1749, + "nearest_rank_p50_us": 1837, + "nearest_rank_p95_us": 2079, + "reported_estimated_inference_time_us": 1749, + "reported_peak_memory_bytes": 90877952, + "compute_unit_row_counts": { + "NPU": 73 + } + }, + "limits": [ + "No task-level numerical tolerance or next-token quality gate is established.", + "Hub component jobs do not prove device-local prefill, KV/ring update, sampling, continuous generation or Omni deployment.", + "Profile sample latency is one component, not complete-request or sustained thermal performance." + ] +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/device_output.npz b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/device_output.npz new file mode 100644 index 000000000..f00cd218e Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/device_output.npz differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/inference_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/inference_report.json new file mode 100644 index 000000000..951c52932 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/inference_report.json @@ -0,0 +1,32 @@ +{ + "timestamp_utc": "2026-09-23T21:38:12.406098+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jp1nj7j2g", + "url": "https://workbench.aihub.qualcomm.com/jobs/jp1nj7j2g/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "SA8775P ADP", + "os": "14", + "attributes": [ + "os:android", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-android", + "format:auto", + "chipset:qualcomm-sa8775p", + "hexagon:v73", + "soc-model:52", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "model_id": "mq9y70k0n", + "input_dataset_id": "d7m83gjy2", + "outputs": "device_output.npz" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/inference_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/inference_submission.json new file mode 100644 index 000000000..667fcf80d --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/inference_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-23T21:26:58.289867+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jpe7z8l15", + "target_model_id": "mq9y70k0n", + "input_dataset_id": "d7m83gjy2", + "device": "SA8775P ADP", + "options": "--compute_unit npu", + "kind": "inference", + "status": "SUBMITTED", + "job_id": "jp1nj7j2g", + "url": "https://workbench.aihub.qualcomm.com/jobs/jp1nj7j2g/" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/profile_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/profile_report.json new file mode 100644 index 000000000..0e64845f0 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/profile_report.json @@ -0,0 +1,751 @@ +{ + "timestamp_utc": "2026-09-23T21:38:17.144484+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jgdd383eg", + "url": "https://workbench.aihub.qualcomm.com/jobs/jgdd383eg/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "SA8775P ADP", + "os": "14", + "attributes": [ + "os:android", + "framework:tflite", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-android", + "format:auto", + "chipset:qualcomm-sa8775p", + "hexagon:v73", + "soc-model:52", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "model_id": "mq9y70k0n", + "shapes": { + "x": [ + [ + 1, + 1, + 2048 + ], + "uint16" + ], + "v_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint16" + ], + "cos_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint16" + ], + "sin_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint16" + ], + "k_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint16" + ], + "mask_full": [ + [ + 1, + 1, + 1, + 1025 + ], + "uint16" + ] + }, + "profile": { + "execution_summary": { + "estimated_inference_time": 1749, + "estimated_inference_peak_memory": 90877952, + "first_load_time": 3647570, + "first_load_peak_memory": 259239936, + "warm_load_time": 382261, + "warm_load_peak_memory": 97042432, + "compile_time": 0, + "compile_peak_memory": 0, + "compile_memory_increase_range": [ + 0, + 0 + ], + "compile_memory_peak_range": [ + 0, + 0 + ], + "first_load_memory_increase_range": [ + 33965328, + 40029184 + ], + "first_load_memory_peak_range": [ + 161763328, + 167827184 + ], + "warm_load_memory_increase_range": [ + 0, + 1755728 + ], + "warm_load_memory_peak_range": [ + 720896, + 87597344 + ], + "inference_memory_increase_range": [ + 0, + 2117632 + ], + "inference_memory_peak_range": [ + 765952, + 86432400 + ], + "all_compile_times": [], + "all_first_load_times": [ + 3647570 + ], + "all_warm_load_times": [ + 382261 + ], + "all_inference_times": [ + 3082, + 1834, + 1909, + 1877, + 1852, + 1835, + 1795, + 1784, + 1809, + 1816, + 1785, + 2063, + 1781, + 1805, + 1800, + 1833, + 1828, + 1808, + 1819, + 1836, + 1903, + 1841, + 1806, + 1913, + 1878, + 1873, + 1875, + 1908, + 1836, + 1911, + 1875, + 1859, + 1893, + 1812, + 1888, + 1825, + 2079, + 1934, + 1920, + 2208, + 2393, + 1855, + 1821, + 1882, + 1823, + 1827, + 1829, + 1929, + 1886, + 1865, + 1939, + 1855, + 1828, + 1778, + 1775, + 1792, + 1811, + 1779, + 1770, + 1810, + 1829, + 1842, + 2060, + 2285, + 1803, + 1801, + 1810, + 1790, + 1793, + 1780, + 1777, + 2510, + 2044, + 1871, + 1837, + 1897, + 1866, + 1845, + 1854, + 1809, + 1834, + 1776, + 1855, + 1882, + 1904, + 1847, + 1822, + 1822, + 1816, + 1965, + 1849, + 1785, + 1749, + 1848, + 1849, + 1871, + 1837, + 1840, + 1884, + 1798 + ] + }, + "execution_detail": [ + { + "name": "Input", + "type": "Input", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 11247 + }, + { + "name": "/layers.0/Pow", + "type": "/layers.0/Pow", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 33404 + }, + { + "name": "/layers.0/ReduceMean", + "type": "/layers.0/ReduceMean", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 9730 + }, + { + "name": "/layers.0/Add", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 8895 + }, + { + "name": "/layers.0/Sqrt", + "type": "/layers.0/Sqrt", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Div", + "type": "/layers.0/Div", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 89939 + }, + { + "name": "/layers.0/Mul", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 65476 + }, + { + "name": "/layers.0/Mul_1", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 9537 + }, + { + "name": "/layers.0/q_k_v_proj/MatMul", + "type": "/layers.0/q", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/q_k_v_proj/MatMul_post_reshape", + "type": "/layers.0/q", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/g_proj/MatMul", + "type": "/layers.0/g", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 7303 + }, + { + "name": "/layers.0/g_proj/MatMul_post_reshape", + "type": "/layers.0/g", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "node_Split_1_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 58156 + }, + { + "name": "node_Split_1_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 21341 + }, + { + "name": "node_Split_1_to_slice_2", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 10062 + }, + { + "name": "/layers.0/Sigmoid", + "type": "/layers.0/Sigmoid", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_1", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_2", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 8808 + }, + { + "name": "/layers.0/Reshape_3", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 16083 + }, + { + "name": "node_Split_3_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 454 + }, + { + "name": "node_Split_3_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1329 + }, + { + "name": "node_Split_5_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 285 + }, + { + "name": "node_Split_5_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 6562 + }, + { + "name": "/layers.0/Concat_5", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 33224 + }, + { + "name": "node_Split_7_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 765 + }, + { + "name": "node_Split_7_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 520 + }, + { + "name": "/layers.0/Mul_2", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 4049 + }, + { + "name": "node_Split_9_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5377 + }, + { + "name": "node_Split_9_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 905 + }, + { + "name": "/layers.0/Mul_4", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5969 + }, + { + "name": "/layers.0/Neg", + "type": "/layers.0/Neg", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 744 + }, + { + "name": "/layers.0/Neg_1", + "type": "/layers.0/Neg", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5334 + }, + { + "name": "/layers.0/Concat", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 697 + }, + { + "name": "/layers.0/Concat_2", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1658 + }, + { + "name": "/layers.0/Mul_3", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3289 + }, + { + "name": "/layers.0/Mul_5", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2271 + }, + { + "name": "/layers.0/Add_1", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 948 + }, + { + "name": "/layers.0/Add_2", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2997 + }, + { + "name": "/layers.0/Concat_1", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 803 + }, + { + "name": "/layers.0/Concat_3", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1305 + }, + { + "name": "/layers.0/Reshape_4", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 325 + }, + { + "name": "/layers.0/Concat_4", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Transpose", + "type": "/layers.0/Transpose", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/MatMul", + "type": "/layers.0/MatMul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 745909 + }, + { + "name": "/layers.0/Mul_6", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2748 + }, + { + "name": "/layers.0/Add_3", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 25552 + }, + { + "name": "/layers.0/Softmax", + "type": "/layers.0/Softmax", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 10411 + }, + { + "name": "/layers.0/MatMul_1", + "type": "/layers.0/MatMul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 651695 + }, + { + "name": "/layers.0/Reshape_5", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_7", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 14796 + }, + { + "name": "/layers.0/out_proj/MatMul", + "type": "/layers.0/out", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 32941 + }, + { + "name": "/layers.0/out_proj/MatMul_post_reshape", + "type": "/layers.0/out", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_4", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 8452 + }, + { + "name": "/layers.0/Pow_1", + "type": "/layers.0/Pow", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5886 + }, + { + "name": "/layers.0/ReduceMean_1", + "type": "/layers.0/ReduceMean", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2501 + }, + { + "name": "/layers.0/Add_5", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 197 + }, + { + "name": "/layers.0/Sqrt_1", + "type": "/layers.0/Sqrt", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Div_1", + "type": "/layers.0/Div", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 58344 + }, + { + "name": "/layers.0/Mul_8", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 38272 + }, + { + "name": "/layers.0/Mul_9", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 7562 + }, + { + "name": "/layers.0/gate_proj/MatMul", + "type": "/layers.0/gate", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/gate_proj/MatMul_post_reshape", + "type": "/layers.0/gate", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/up_proj/MatMul", + "type": "/layers.0/up", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 414626 + }, + { + "name": "/layers.0/up_proj/MatMul_post_reshape", + "type": "/layers.0/up", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "_elementwiseneuron_0", + "type": "elementwiseneuron", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 396528 + }, + { + "name": "/layers.0/Mul_12", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 21386 + }, + { + "name": "/layers.0/down_proj/MatMul", + "type": "/layers.0/down", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 369308 + }, + { + "name": "/layers.0/down_proj/MatMul_post_reshape", + "type": "/layers.0/down", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_7", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5980 + }, + { + "name": "/layers.0/Concat_3_identity", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_2_identity", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "Output", + "type": "Output", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 7279 + } + ] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/profile_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/profile_submission.json new file mode 100644 index 000000000..1fb8616b7 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_sa8775_full_attention/profile_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-23T21:26:59.698671+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jpe7z8l15", + "target_model_id": "mq9y70k0n", + "input_dataset_id": null, + "device": "SA8775P ADP", + "options": "--compute_unit npu", + "kind": "profile", + "status": "SUBMITTED", + "job_id": "jgdd383eg", + "url": "https://workbench.aihub.qualcomm.com/jobs/jgdd383eg/" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md new file mode 100644 index 000000000..becc0709b --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md @@ -0,0 +1,13 @@ +# Spark-X2.5 attention layer on Snapdragon X Elite CRD + +**One fixed-shape decoder attention layer from the real Spark-X2.5-1.7B checkpoint executed on the hosted Snapdragon X Elite CRD NPU (Windows 11 ARM64). This is component evidence, not complete Spark generation or a local Omni deployment.** + +The [Galaxy S24 provenance record](../spark_s24_full_attention/README.md) pins the FP32 source ONNX, W8A16 QDQ ONNX, calibration and exact input fixture. Its QNN DLC compile job `jpe7z8l15` produced target model `mq9y70k0n`; the same DLC and fixture dataset `d7m83gjy2` were reused here without recompilation. [Inference submission](inference_submission.json) job `j5wl7876p` and [profile submission](profile_submission.json) job `jg9zmkmlp` both requested `--compute_unit npu` for the exact device. The [raw inference job](inference_report.json) and [raw profile job](profile_report.json) completed. Their service build, OS, chipset and target IDs are preserved in those reports. + +The [three outputs](device_output.npz) were finite FP32 hidden/new-key/new-value tensors at a 1,024-token cache shape. They were bitwise identical to the earlier Galaxy S25 and S24 NPU outputs on the same fixture. The [audit](audit_report.json) measured hidden/key/value relative L2 of **1.351%/0.377%/0.650%** against original FP32 ONNX CPU, and **0.555%/0.0065%/0.0126%** against QDQ ONNX CPU. No next-token or task-level quality tolerance has been established. + +All 73 execution-detail rows in the device profile were attributed to NPU. The 100 component inference-time samples had nearest-rank p50/p95 **1.433/1.606 ms** (minimum 1.346 ms). The service's `estimated_inference_time` was 1.346 ms, matching the minimum rather than the median. The reported component inference peak memory was 18,423,808 bytes. These are hosted component-job measurements; they exclude prefill, 27 other decoder layers, sampling, transfer, loading and a resident KV/ring/controller. Hardware, OS and hosted power state differ from the mobile runs, so these are not paired speed comparisons. + +Re-audit with `benchmarks/edge_harness/experiments/audit_spark_qualcomm_attention.py` and the [pinned source artifact paths](../spark_s24_full_attention/README.md). `benchmarks/edge_harness/experiments/collect_spark_aihub_component.py` retrieves the existing jobs using the local AI Hub client configuration. Neither script contains credentials. + +**Matrix status: component NPU inference/profiled; NOT E2E.** Full qualification still needs an X Elite machine running the complete 28-layer Spark model through a resident Omni backend with prefill/decode/KV-ring/sampling, state and cancellation, token-quality checks, memory admission and sustained power/thermal measurements. Workbench is a test facility, not that deployment runtime. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/audit_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/audit_report.json new file mode 100644 index 000000000..5045ec833 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/audit_report.json @@ -0,0 +1,142 @@ +{ + "scope": "one real Spark-X2.5 full-attention decoder layer at one static 1024-token cache shape; no generation E2E", + "status": "component_executed_on_npu_quality_unqualified", + "device": { + "name": "Snapdragon X Elite CRD", + "os": "11", + "attributes": [ + "os:windows", + "format:compute", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-windows", + "chipset:qualcomm-snapdragon-x-elite", + "chipset:sc8380xp", + "hexagon:v73", + "soc-model:60", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "compile_job_id": "jpe7z8l15", + "inference_job_id": "j5wl7876p", + "profile_job_id": "jg9zmkmlp", + "target_model_id": "mq9y70k0n", + "input_dataset_id": "d7m83gjy2", + "s25_prior_inference_job_id": "jp0mvkj6g", + "files": { + "source_onnx": { + "bytes": 205618089, + "sha256": "4e2b1c8d791b1ee0e9b75f3322a20a743c0bf8c40f9b98edd963f2944d6300bb" + }, + "quantized_onnx_zip": { + "bytes": 186772390, + "sha256": "703821ca8ace5672228919ea2add4901def4887748c381b12ffc8b259665c553" + }, + "fixture": { + "bytes": 3901238, + "sha256": "5a439e6c6d148f1720938770daa110039a39edde014f1c1592c823c7d7dfc8ed" + }, + "source_cpu_output": { + "bytes": 12103, + "sha256": "6bbfb8e6ccb09475391ce1dbd2e855205720f4367c038a0d5be35e7dfeb2f768" + }, + "quantized_cpu_output": { + "bytes": 11574, + "sha256": "061654773166d102a11228c4a2e4cd06de6c2482c7e4e7860315b7b9a95fa8a1" + }, + "s25_output": { + "bytes": 13072, + "sha256": "2dea9f4b478cb02b9be74a8d3ea4ad42e562e4de72413967d22706a1b6c6b9c0" + }, + "device_output": { + "bytes": 11576, + "sha256": "9381fccee14b303edc5b432edf92422f2948e111ea6848df22cba3cd289a7073" + } + }, + "source_vs_quantized_cpu": { + "hidden": { + "relative_l2": 0.012364819167814612, + "max_abs": 0.0028282999992370605, + "cosine": 0.9999236499353131 + }, + "k_new_0": { + "relative_l2": 0.0037627070087598766, + "max_abs": 0.03641480207443237, + "cosine": 0.9999929226489408 + }, + "v_new_0": { + "relative_l2": 0.006507432119585315, + "max_abs": 0.011739224195480347, + "cosine": 0.9999789079393114 + } + }, + "quantized_cpu_vs_device_npu": { + "hidden": { + "relative_l2": 0.005550082388164841, + "max_abs": 0.002016603946685791, + "cosine": 0.9999847274053486 + }, + "k_new_0": { + "relative_l2": 6.533750243101402e-05, + "max_abs": 0.00033855438232421875, + "cosine": 0.9999999978819449 + }, + "v_new_0": { + "relative_l2": 0.0001257028618212242, + "max_abs": 0.0002911090850830078, + "cosine": 0.9999999921460195 + } + }, + "source_cpu_vs_device_npu": { + "hidden": { + "relative_l2": 0.01350919207884596, + "max_abs": 0.0048449039459228516, + "cosine": 0.9999091967737982 + }, + "k_new_0": { + "relative_l2": 0.003766040678719007, + "max_abs": 0.03641480207443237, + "cosine": 0.9999929097855774 + }, + "v_new_0": { + "relative_l2": 0.006504874746412689, + "max_abs": 0.011739224195480347, + "cosine": 0.9999789285134126 + } + }, + "s25_npu_vs_device_npu": { + "hidden": { + "relative_l2": 0.0, + "max_abs": 0.0, + "cosine": 0.9999999999999998 + }, + "k_new_0": { + "relative_l2": 0.0, + "max_abs": 0.0, + "cosine": 0.9999999999999998 + }, + "v_new_0": { + "relative_l2": 0.0, + "max_abs": 0.0, + "cosine": 1.0 + } + }, + "profile": { + "sample_count": 100, + "sample_min_us": 1346, + "nearest_rank_p50_us": 1433, + "nearest_rank_p95_us": 1606, + "reported_estimated_inference_time_us": 1346, + "reported_peak_memory_bytes": 18423808, + "compute_unit_row_counts": { + "NPU": 73 + } + }, + "limits": [ + "No task-level numerical tolerance or next-token quality gate is established.", + "Hub component jobs do not prove device-local prefill, KV/ring update, sampling, continuous generation or Omni deployment.", + "Profile sample latency is one component, not complete-request or sustained thermal performance." + ] +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/device_output.npz b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/device_output.npz new file mode 100644 index 000000000..f00cd218e Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/device_output.npz differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/inference_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/inference_report.json new file mode 100644 index 000000000..5a99ee90f --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/inference_report.json @@ -0,0 +1,32 @@ +{ + "timestamp_utc": "2026-09-23T21:33:23.832588+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "j5wl7876p", + "url": "https://workbench.aihub.qualcomm.com/jobs/j5wl7876p/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Snapdragon X Elite CRD", + "os": "11", + "attributes": [ + "os:windows", + "format:compute", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-windows", + "chipset:qualcomm-snapdragon-x-elite", + "chipset:sc8380xp", + "hexagon:v73", + "soc-model:60", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "model_id": "mq9y70k0n", + "input_dataset_id": "d7m83gjy2", + "outputs": "device_output.npz" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/inference_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/inference_submission.json new file mode 100644 index 000000000..ac7e9d020 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/inference_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-23T21:26:55.958764+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jpe7z8l15", + "target_model_id": "mq9y70k0n", + "input_dataset_id": "d7m83gjy2", + "device": "Snapdragon X Elite CRD", + "options": "--compute_unit npu", + "kind": "inference", + "status": "SUBMITTED", + "job_id": "j5wl7876p", + "url": "https://workbench.aihub.qualcomm.com/jobs/j5wl7876p/" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/profile_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/profile_report.json new file mode 100644 index 000000000..2692f87e6 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/profile_report.json @@ -0,0 +1,751 @@ +{ + "timestamp_utc": "2026-09-23T21:33:27.915381+00:00", + "qai_hub_version": "0.55.0", + "facility": "Qualcomm AI Hub Workbench", + "job_id": "jg9zmkmlp", + "url": "https://workbench.aihub.qualcomm.com/jobs/jg9zmkmlp/", + "status": "SUCCESS", + "status_message": "", + "options": "--compute_unit npu", + "hub_version": "aihub-2026.09.11.0", + "device": { + "name": "Snapdragon X Elite CRD", + "os": "11", + "attributes": [ + "os:windows", + "format:compute", + "framework:onnx", + "vendor:qualcomm", + "abi:aarch64-windows", + "chipset:qualcomm-snapdragon-x-elite", + "chipset:sc8380xp", + "hexagon:v73", + "soc-model:60", + "htp-supports-fp16:true", + "htp-supports-weight-sharing:true", + "framework:qnn" + ] + }, + "model_id": "mq9y70k0n", + "shapes": { + "x": [ + [ + 1, + 1, + 2048 + ], + "uint16" + ], + "v_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint16" + ], + "cos_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint16" + ], + "sin_full": [ + [ + 1, + 1, + 1, + 64 + ], + "uint16" + ], + "k_cache_0": [ + [ + 1, + 2, + 1024, + 256 + ], + "uint16" + ], + "mask_full": [ + [ + 1, + 1, + 1, + 1025 + ], + "uint16" + ] + }, + "profile": { + "execution_summary": { + "estimated_inference_time": 1346, + "estimated_inference_peak_memory": 18423808, + "first_load_time": 2429828, + "first_load_peak_memory": 318615552, + "warm_load_time": 242574, + "warm_load_peak_memory": 14589952, + "compile_time": 0, + "compile_peak_memory": 0, + "compile_memory_increase_range": [ + 0, + 0 + ], + "compile_memory_peak_range": [ + 0, + 0 + ], + "first_load_memory_increase_range": [ + 207298560, + 207298560 + ], + "first_load_memory_peak_range": [ + 313659392, + 313659392 + ], + "warm_load_memory_increase_range": [ + 9183232, + 9183232 + ], + "warm_load_memory_peak_range": [ + 9629696, + 9629696 + ], + "inference_memory_increase_range": [ + 20480, + 20480 + ], + "inference_memory_peak_range": [ + 2109440, + 2109440 + ], + "all_compile_times": [], + "all_first_load_times": [ + 2429828 + ], + "all_warm_load_times": [ + 242574 + ], + "all_inference_times": [ + 2684, + 1456, + 1479, + 1446, + 1378, + 1375, + 1481, + 1412, + 1430, + 1431, + 1540, + 1399, + 1398, + 1526, + 1439, + 1455, + 1599, + 1416, + 1352, + 1433, + 1440, + 1507, + 1472, + 1685, + 1546, + 1512, + 1531, + 1468, + 1353, + 1367, + 1454, + 1434, + 1489, + 1514, + 1421, + 1364, + 1387, + 1414, + 1390, + 1412, + 1398, + 1413, + 1429, + 1549, + 1467, + 1425, + 1525, + 1420, + 1523, + 1606, + 1504, + 1490, + 1494, + 1458, + 1521, + 1410, + 1733, + 1548, + 1398, + 1494, + 1426, + 1449, + 1389, + 1352, + 1415, + 1501, + 1564, + 1407, + 1548, + 1461, + 1531, + 1505, + 1514, + 1690, + 1451, + 1488, + 3416, + 1418, + 1375, + 1411, + 1416, + 1545, + 1425, + 1409, + 1406, + 1419, + 1405, + 1383, + 1429, + 1381, + 1467, + 1396, + 1418, + 1361, + 1346, + 1365, + 1452, + 1408, + 1390, + 1409 + ] + }, + "execution_detail": [ + { + "name": "Input", + "type": "Input", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 15300 + }, + { + "name": "/layers.0/Pow", + "type": "/layers.0/Pow", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 41331 + }, + { + "name": "/layers.0/ReduceMean", + "type": "/layers.0/ReduceMean", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 6753 + }, + { + "name": "/layers.0/Add", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 7094 + }, + { + "name": "/layers.0/Sqrt", + "type": "/layers.0/Sqrt", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Div", + "type": "/layers.0/Div", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 66952 + }, + { + "name": "/layers.0/Mul", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 59984 + }, + { + "name": "/layers.0/Mul_1", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 12187 + }, + { + "name": "/layers.0/q_k_v_proj/MatMul", + "type": "/layers.0/q", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/q_k_v_proj/MatMul_post_reshape", + "type": "/layers.0/q", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/g_proj/MatMul", + "type": "/layers.0/g", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5297 + }, + { + "name": "/layers.0/g_proj/MatMul_post_reshape", + "type": "/layers.0/g", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "node_Split_1_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 34704 + }, + { + "name": "node_Split_1_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 17816 + }, + { + "name": "node_Split_1_to_slice_2", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 8512 + }, + { + "name": "/layers.0/Sigmoid", + "type": "/layers.0/Sigmoid", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_1", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_2", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 8560 + }, + { + "name": "/layers.0/Reshape_3", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 7452 + }, + { + "name": "node_Split_3_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 319 + }, + { + "name": "node_Split_3_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1721 + }, + { + "name": "node_Split_5_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 278 + }, + { + "name": "node_Split_5_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 6044 + }, + { + "name": "/layers.0/Concat_5", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 29616 + }, + { + "name": "node_Split_7_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 989 + }, + { + "name": "node_Split_7_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 525 + }, + { + "name": "/layers.0/Mul_2", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3468 + }, + { + "name": "node_Split_9_to_slice_0", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1018 + }, + { + "name": "node_Split_9_to_slice_1", + "type": "node", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1371 + }, + { + "name": "/layers.0/Mul_4", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 9373 + }, + { + "name": "/layers.0/Neg", + "type": "/layers.0/Neg", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 266 + }, + { + "name": "/layers.0/Neg_1", + "type": "/layers.0/Neg", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3136 + }, + { + "name": "/layers.0/Concat", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 817 + }, + { + "name": "/layers.0/Concat_2", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2424 + }, + { + "name": "/layers.0/Mul_3", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3893 + }, + { + "name": "/layers.0/Mul_5", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2650 + }, + { + "name": "/layers.0/Add_1", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 361 + }, + { + "name": "/layers.0/Add_2", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 2670 + }, + { + "name": "/layers.0/Concat_1", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1245 + }, + { + "name": "/layers.0/Concat_3", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1124 + }, + { + "name": "/layers.0/Reshape_4", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 362 + }, + { + "name": "/layers.0/Concat_4", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Transpose", + "type": "/layers.0/Transpose", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/MatMul", + "type": "/layers.0/MatMul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 746665 + }, + { + "name": "/layers.0/Mul_6", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 3374 + }, + { + "name": "/layers.0/Add_3", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 17560 + }, + { + "name": "/layers.0/Softmax", + "type": "/layers.0/Softmax", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 14098 + }, + { + "name": "/layers.0/MatMul_1", + "type": "/layers.0/MatMul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 658338 + }, + { + "name": "/layers.0/Reshape_5", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Mul_7", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 32722 + }, + { + "name": "/layers.0/out_proj/MatMul", + "type": "/layers.0/out", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 35771 + }, + { + "name": "/layers.0/out_proj/MatMul_post_reshape", + "type": "/layers.0/out", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_4", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 9773 + }, + { + "name": "/layers.0/Pow_1", + "type": "/layers.0/Pow", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 4443 + }, + { + "name": "/layers.0/ReduceMean_1", + "type": "/layers.0/ReduceMean", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 1724 + }, + { + "name": "/layers.0/Add_5", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 184 + }, + { + "name": "/layers.0/Sqrt_1", + "type": "/layers.0/Sqrt", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Div_1", + "type": "/layers.0/Div", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 50440 + }, + { + "name": "/layers.0/Mul_8", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 36313 + }, + { + "name": "/layers.0/Mul_9", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 8408 + }, + { + "name": "/layers.0/gate_proj/MatMul", + "type": "/layers.0/gate", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/gate_proj/MatMul_post_reshape", + "type": "/layers.0/gate", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/up_proj/MatMul", + "type": "/layers.0/up", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 301316 + }, + { + "name": "/layers.0/up_proj/MatMul_post_reshape", + "type": "/layers.0/up", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "_elementwiseneuron_0", + "type": "elementwiseneuron", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 319204 + }, + { + "name": "/layers.0/Mul_12", + "type": "/layers.0/Mul", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 20213 + }, + { + "name": "/layers.0/down_proj/MatMul", + "type": "/layers.0/down", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 312506 + }, + { + "name": "/layers.0/down_proj/MatMul_post_reshape", + "type": "/layers.0/down", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Add_7", + "type": "/layers.0/Add", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 5976 + }, + { + "name": "/layers.0/Concat_3_identity", + "type": "/layers.0/Concat", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "/layers.0/Reshape_2_identity", + "type": "/layers.0/Reshape", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 0 + }, + { + "name": "Output", + "type": "Output", + "compute_unit": "NPU", + "execution_time": 0, + "execution_cycles": 15482 + } + ] + } +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/profile_submission.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/profile_submission.json new file mode 100644 index 000000000..11f831f2d --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_xelite_full_attention/profile_submission.json @@ -0,0 +1,14 @@ +{ + "timestamp_utc": "2026-09-23T21:26:57.092534+00:00", + "facility": "Qualcomm AI Hub Workbench", + "qai_hub_version": "0.55.0", + "source_compile_job_id": "jpe7z8l15", + "target_model_id": "mq9y70k0n", + "input_dataset_id": null, + "device": "Snapdragon X Elite CRD", + "options": "--compute_unit npu", + "kind": "profile", + "status": "SUBMITTED", + "job_id": "jg9zmkmlp", + "url": "https://workbench.aihub.qualcomm.com/jobs/jg9zmkmlp/" +} diff --git a/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md b/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md index 5b189bae5..12dd205f0 100644 --- a/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md +++ b/benchmarks/edge_harness/results/e2e_profiling_20260922/evidence/summary/README.md @@ -7,6 +7,10 @@ The WSL CPU and CPU+Radeon InternVLA cells also have [bounded Omni complete-poli A [follow-up MiniCPM-o storage experiment](../../../e2e_expansion_20260923/evidence/minicpmo_omni_native_weights_io/README.md) copied the same ten GGUF weights to native NTFS. One warmup plus 20 complete requests passed on Windows CPU (p50/p95 23.62/23.88 s) and Radeon+CPU (16.35/17.20 s), with no prefetch warnings. Separate one-request UNC controls were much slower; the original UNC 20-request Radeon profile remains valid for that path. Run order and OS cache were not controlled, so no causal or cross-placement speedup claim follows. A [real InternVLA Cosmos encoder NPU artifact attempt](../../../e2e_expansion_20260923/evidence/internvla_cosmos_amd_npu/README.md) passed fixed-shape CPU export parity but its tested A16W8 candidates failed the CPU numerical gate or VitisAI compilation; the AMD NPU cell remains NOT E2E. The [WSL CPU Spark GGUF Omni profile](../../../e2e_expansion_20260923/evidence/spark_omni_wsl_cpu/README.md) adds a separate current-Omni complete-request path to the earlier vLLM INT8 CPU result. It passed two named prompts, 20/20 serial measured inventory requests at p50/p95 7.82/7.97 s, a public text request, context refusal and cancellation cleanup. The CPU-only llama.cpp build required explicit CPU-buffer placement evidence because it omits the Windows offload-count log line. This does not establish token streaming or broad quality. + +On a hosted [Galaxy S24 Spark attention-layer experiment](../../../e2e_expansion_20260923/evidence/spark_s24_full_attention/README.md), a real one-layer W8A16 QDQ artifact executed on NPU and matched prior S25 component outputs exactly on one dataset. The 100-sample layer p50/p95 was 1.109/1.205 ms; the service's 1.098 ms estimated-time field was the sample minimum. FP32-source versus S24 output hidden-state relative L2 was 1.351% without a downstream token-quality gate. This adds component C evidence only; device-local generation remains absent. + +The same W8A16 layer also ran on hosted [Snapdragon X Elite CRD](../../../e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md) and [SA8775P ADP](../../../e2e_expansion_20260923/evidence/spark_sa8775_full_attention/README.md) NPUs, with pinned-fixture outputs identical to S24/S25; the 100-sample component p50/p95 was 1.433/1.606 ms and 1.837/2.079 ms, respectively. On [RB3 Gen 2 / QCS6490](../../../e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md), that W8A16 graph failed QNN DLC loading even after target-specific recompilation. An FP32 ONNX CPU layer then passed with source-relative L2 below 9e-7, while a calibrated W8A8 NPU layer executed but had 20.9% hidden-state relative L2 and remains numerically unqualified. These hosted component outcomes do not establish full generation or comparable device speed. ## Configuration × model @@ -27,11 +31,11 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i | PC HX370 + Radeon 890M / Windows worker | scoped E2E: Omni Spark text | scoped E2E: Omni Radeon talker/codec + CPU predictor | NOT E2E | scoped E2E: Omni Radeon+CPU image+audio → text+speech (native-storage 20-run profile) | synthetic policy pass: Omni CPU+Radeon Cosmos (WSL/native Windows) | | PC HX370 + AMD NPU / Windows worker | NOT E2E (output-head component passed) | NOT E2E | NOT E2E | NOT E2E | NOT E2E | | PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E | -| PC Snapdragon X Elite CRD / AI Hub | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E | +| PC Snapdragon X Elite CRD / AI Hub | NOT E2E (one Spark attention layer on NPU) | NOT E2E | NOT E2E | NOT E2E | NOT E2E | | Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E | -| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E | -| Embedded SA8775P ADP / AI Hub | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E | -| Embedded RB3 Gen 2 / QCS6490 / AI Hub | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E | +| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | NOT E2E (one Spark attention layer on NPU) | NOT E2E | NOT E2E | NOT E2E | NOT E2E | +| Embedded SA8775P ADP / AI Hub | NOT E2E (one Spark attention layer on NPU) | NOT E2E | NOT E2E | NOT E2E | NOT E2E | +| Embedded RB3 Gen 2 / QCS6490 / AI Hub | NOT E2E (Spark CPU layer passed; W8A8 NPU quality open) | NOT E2E | NOT E2E | NOT E2E | NOT E2E | ## Per-cell reasons and next steps @@ -72,7 +76,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i | PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | InternVLA-A1 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | -| PC Snapdragon X Elite CRD / AI Hub | Spark-X2.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | +| PC Snapdragon X Elite CRD / AI Hub | Spark-X2.5 | COMPONENT_NPU_INFERENCE_PROFILED; NOT_E2E_PROFILEABLE | A real Spark-X2.5-1.7B one-layer W8A16 QDQ attention artifact, compiled as QNN DLC, ran on the exact hosted Snapdragon X Elite CRD (Windows 11 ARM64). Inference returned finite hidden/new-K/new-V tensors bitwise identical to the prior S24/S25 NPU outputs on the same fixture; the profile attributed all 73 execution-detail rows to NPU. Across 100 component samples, nearest-rank p50/p95 was 1.433/1.606 ms; the 1.346 ms estimated-time field is the minimum. Hidden/K/V relative L2 versus original FP32 ONNX CPU was 1.351%/0.377%/0.650%, without a next-token quality tolerance. There is no complete 28-layer prefill/decode/KV-ring/sampling loop, resident Omni controller or complete-request timing. | Qualify a complete X Elite resident artifact/controller with state, token quality, memory admission/cancel and sustained power; [X Elite NPU raw evidence](../../../e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md). | | PC Snapdragon X Elite CRD / AI Hub | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | Historical QNN talker profile re-fetched successfully; no numerical/full-pipeline gate. Historical TFLite vocoder target was refused on this Windows ARM device. | Validate target-specific numerical quality, complete local execution, memory and sustained behavior. | | PC Snapdragon X Elite CRD / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | PC Snapdragon X Elite CRD / AI Hub | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | @@ -82,17 +86,17 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i | Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | Fresh FP16 speech-head inference passed; all 41 outputs exactly match the same artifact/dataset reference. Historical FP16 numerical parity passed; tested W8A16 top-1 parity failed. Thinker/encoders/vocoder are not qualified together. | Supply a qualified complete mobile checkpoint/export and validate the full multimodal/stateful pipeline. | | Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | -| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | Spark-X2.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | +| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | Spark-X2.5 | COMPONENT_NPU_INFERENCE_PROFILED; NOT_E2E_PROFILEABLE | The real 1.7B checkpoint's fixed-shape full-attention decoder **one-layer** W8A16 QDQ artifact compiled as QNN DLC and ran on an exact hosted Galaxy S24 (Android 14). A same-dataset inference returned finite hidden/new-K/new-V outputs identical to the prior S25 NPU outputs; the profile attributed all 73 execution-detail rows to NPU. Across 100 component samples, nearest-rank p50/p95 was 1.109/1.205 ms; the 1.098 ms `estimated_inference_time` field is the minimum, not a median. Relative L2 versus original FP32 ONNX CPU was 1.351%/0.377%/0.650% for hidden/K/V; no token-quality tolerance is known. This is one cache bucket, not 28-layer prefill/decode, KV-ring, sampling, a resident local controller or complete-request timing. ADB has no attached device. | Qualify a complete device-resident artifact/controller with state and token-quality gates, 128-token generation across 512/1024 context transitions, admission/cancel and sustained memory/power; [S24 NPU raw evidence](../../../e2e_expansion_20260923/evidence/spark_s24_full_attention/README.md). | | Mobile Galaxy S24 / Snapdragon 8 Gen 3 | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | Historical FP16 decoder parity has SNR 39.13 dB. Tested calibrated INT8 decoder parity failed with negative SNR. Neither result is a full local TTS stream. | Validate the complete local pipeline and a quality-passing artifact set. | | Mobile Galaxy S24 / Snapdragon 8 Gen 3 | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | Mobile Galaxy S24 / Snapdragon 8 Gen 3 | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | Historical FP16 speech-head job re-fetched successfully; SNR 49.22 dB/top-1 match in the recorded parity test. W8A16 top-1 failed. Not complete MiniCPM-o. | Qualify thinker/encoders/vocoder and stateful local integration. | | Mobile Galaxy S24 / Snapdragon 8 Gen 3 | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | -| Embedded SA8775P ADP / AI Hub | Spark-X2.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | +| Embedded SA8775P ADP / AI Hub | Spark-X2.5 | COMPONENT_NPU_INFERENCE_PROFILED; NOT_E2E_PROFILEABLE | A real Spark-X2.5-1.7B one-layer W8A16 QDQ attention artifact, compiled as QNN DLC, ran on the exact hosted SA8775P ADP (Android 14). Inference returned finite hidden/new-K/new-V tensors bitwise identical to the prior S24/S25 NPU outputs on the same fixture; the profile attributed all 73 execution-detail rows to NPU. Across 100 component samples, nearest-rank p50/p95 was 1.837/2.079 ms; the 1.749 ms estimated-time field is the minimum. Hidden/K/V relative L2 versus original FP32 ONNX CPU was 1.351%/0.377%/0.650%, without a next-token quality tolerance. There is no complete 28-layer prefill/decode/KV-ring/sampling loop, resident Omni controller or complete-request timing. | Qualify a complete embedded resident artifact/controller with state, token quality, memory admission/cancel and sustained power; [SA8775P NPU raw evidence](../../../e2e_expansion_20260923/evidence/spark_sa8775_full_attention/README.md). | | Embedded SA8775P ADP / AI Hub | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | Historical NPU talker and GPU vocoder profiles re-fetched successfully; no complete pipeline, parity or co-residency qualification. | Validate target-specific numerical quality, complete local execution, memory and sustained behavior. | | Embedded SA8775P ADP / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | Embedded SA8775P ADP / AI Hub | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | Embedded SA8775P ADP / AI Hub | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | -| Embedded RB3 Gen 2 / QCS6490 / AI Hub | Spark-X2.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | +| Embedded RB3 Gen 2 / QCS6490 / AI Hub | Spark-X2.5 | COMPONENT_CPU_PARITY_PASS; COMPONENT_NPU_NUMERIC_UNQUALIFIED; NOT_E2E_PROFILEABLE | On the exact hosted Dragonwing RB3 Gen 2 Vision Kit (QCS6490, QC Linux 1.6), the reused S24 W8A16 QNN DLC and a separately RB3-compiled W8A16 DLC each failed inference/profile at QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR. An FP32 ONNX one-layer CPU route passed with all 56 profile rows on CPU, source-relative hidden/K/V L2 of 8.73e-7/2.98e-7/3.37e-7 and 100-sample component p50/p95 19.304/21.113 ms. A calibrated W8A8 QNN DLC ran with all 61 rows on NPU and p50/p95 5.610/5.761 ms, but hidden/K/V relative L2 was 20.914%/1.702%/4.413% versus FP32 CPU, without a next-token quality gate. These are different precisions, not a qualified speedup; no complete 28-layer prefill/decode/KV-ring/sampling, local Omni controller or complete-request timing exists. | Improve representative W8A8 calibration or find another compatible quality-passing artifact; validate token parity, then full device-resident generation, admission/cancel and sustained power; [RB3 raw evidence](../../../e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md). | | Embedded RB3 Gen 2 / QCS6490 / AI Hub | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | Historical GPU vocoder profile re-fetched successfully. The attempted NPU talker build was rejected for floating-point inputs. No full model pipeline. | Validate target-specific numerical quality, complete local execution, memory and sustained behavior. | | Embedded RB3 Gen 2 / QCS6490 / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | | Embedded RB3 Gen 2 / QCS6490 / AI Hub | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. | diff --git a/docs/design/omni_edge_e2e_roadmap.md b/docs/design/omni_edge_e2e_roadmap.md index 4ec255e51..58001504d 100644 --- a/docs/design/omni_edge_e2e_roadmap.md +++ b/docs/design/omni_edge_e2e_roadmap.md @@ -13,6 +13,8 @@ three-stage text-to-speech functional pass after correcting request embeddings, and an InternVLA real-weight synthetic action forward. Model-quality and device gates remain open; the 2026-09-22 audit remains the historical baseline. +The [Spark hosted Qualcomm component expansion](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/README.md) advances M1 only to one real fixed-shape decoder attention layer. The same W8A16 QNN DLC ran on S24, Snapdragon X Elite CRD and SA8775P ADP NPUs with identical outputs on one pinned fixture; S25 had prior separate component evidence. On [RB3 Gen 2](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md), the tested W8A16 graph failed QNN loading even after an exact-device compile. FP32 ONNX ran on CPU with near-source parity, while calibrated W8A8 ran on NPU but had 20.9% hidden-state relative L2 and remains numerically unqualified. These are component C results, not full M1: device-local 28-layer prefill/continuous decode, KV/ring, sampling, token quality, admission/cancellation, complete-request timing and sustained power/thermal gates remain open. AI Hub is the test facility, not a deployment dependency. + The [WSL CPU expansion](../../benchmarks/edge_harness/results/e2e_expansion_20260923/README.md) adds constrained three-stage MiniCPM-o text-to-speech requests and a CPU-only InternVLA synthetic policy forward. A subsequent 20-request serial MiniCPM-o