Audit mobile speech heads and S25 VLA latent impact
This commit is contained in:
1 parent
1949c31f80
commit
d71d39f656
53 files changed
+3333
-30
No files matched your search
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Audit one MiniCPM-o speech-head decode step against pinned ONNX CPU tensors."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
SOURCE_SHA256 = "78cf64804f11ad269ee4180da61588ccc5eefe2942773227345499756e4cc229"
|
||||
FIXTURE_SHA256 = "65919ea1f351feec33c1415a3e3db76d60bc9c2beb3d49a72af75dcb418e83b4"
|
||||
REFERENCE_SHA256 = "902adcceadbe761d938ef2ade51cf953c8917ea1abfaee33a10c84e300c1bd96"
|
||||
COMPILE_OPTIONS = {
|
||||
"--target_runtime qnn_context_binary --quantize_full_type float16",
|
||||
"--target_runtime qnn_dlc --qnn_options default_graph_htp_precision=FLOAT16",
|
||||
}
|
||||
INPUT_NAMES = ["x", "cos", "sin", "mask"] + [
|
||||
name for layer in range(20) for name in (f"k_cache_{layer}", f"v_cache_{layer}")
|
||||
]
|
||||
OUTPUT_NAMES = ["logits"] + [
|
||||
name for layer in range(20) for name in (f"k_new_{layer}", f"v_new_{layer}")
|
||||
]
|
||||
|
||||
|
||||
def file_record(path: Path) -> dict:
|
||||
with path.open("rb") as source:
|
||||
digest = hashlib.file_digest(source, "sha256").hexdigest()
|
||||
return {"bytes": path.stat().st_size, "sha256": digest}
|
||||
|
||||
|
||||
def read_array(data: np.lib.npyio.NpzFile, name: str, shape: tuple[int, ...]) -> np.ndarray:
|
||||
value = np.asarray(data[name])
|
||||
if value.shape != shape or value.dtype != np.float32 or not np.isfinite(value).all():
|
||||
raise ValueError(f"{name}: expected finite FP32 {shape}, got {value.shape}/{value.dtype}")
|
||||
return value
|
||||
|
||||
|
||||
def compare(reference: np.ndarray, actual: np.ndarray) -> dict:
|
||||
source = reference.astype(np.float64).ravel()
|
||||
observed = actual.astype(np.float64).ravel()
|
||||
error = source - observed
|
||||
source_norm = float(np.linalg.norm(source))
|
||||
error_norm = float(np.linalg.norm(error))
|
||||
return {
|
||||
"relative_l2": error_norm / source_norm if source_norm else (0.0 if error_norm == 0 else None),
|
||||
"max_abs_error": float(np.max(np.abs(error))),
|
||||
"source_rms": float(np.sqrt(np.mean(source * source))),
|
||||
"device_rms": float(np.sqrt(np.mean(observed * observed))),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--device-name", required=True)
|
||||
parser.add_argument("--source-onnx", type=Path, required=True)
|
||||
parser.add_argument("--fixture", type=Path, required=True)
|
||||
parser.add_argument("--cpu-reference", type=Path, required=True)
|
||||
parser.add_argument("--compile-report", type=Path, required=True)
|
||||
parser.add_argument("--inference-report", type=Path, required=True)
|
||||
parser.add_argument("--device-output", type=Path, required=True)
|
||||
parser.add_argument("--output-report", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
compiled = json.loads(args.compile_report.read_text(encoding="utf-8"))
|
||||
inferred = json.loads(args.inference_report.read_text(encoding="utf-8"))
|
||||
if compiled.get("status") != "SUCCESS" or inferred.get("status") != "SUCCESS":
|
||||
raise ValueError("compile and inference must be terminal successes")
|
||||
if compiled.get("source_model_id") != "mn4gjjpvn" or compiled.get("options") not in COMPILE_OPTIONS:
|
||||
raise ValueError("compile source/options differ from the pinned speech-head artifact")
|
||||
if inferred.get("model_id") != compiled.get("target_model_id"):
|
||||
raise ValueError("inference used a different target artifact")
|
||||
if inferred.get("input_dataset_id") not in {"d7gwxe3y2", "d7zn8rk57"}:
|
||||
raise ValueError("inference did not use a retained matching fixture dataset")
|
||||
if (compiled.get("device", {}).get("name") != args.device_name
|
||||
or inferred.get("device", {}).get("name") != args.device_name):
|
||||
raise ValueError("jobs did not use the exact requested device")
|
||||
|
||||
files = {
|
||||
name: file_record(path)
|
||||
for name, path in {
|
||||
"source_onnx": args.source_onnx,
|
||||
"fixture": args.fixture,
|
||||
"cpu_reference": args.cpu_reference,
|
||||
"device_output": args.device_output,
|
||||
}.items()
|
||||
}
|
||||
for name, expected in {
|
||||
"source_onnx": SOURCE_SHA256,
|
||||
"fixture": FIXTURE_SHA256,
|
||||
"cpu_reference": REFERENCE_SHA256,
|
||||
}.items():
|
||||
if files[name]["sha256"] != expected:
|
||||
raise ValueError(f"{name} differs from the pinned artifact")
|
||||
|
||||
with np.load(args.fixture, allow_pickle=False) as fixture:
|
||||
if fixture.files != INPUT_NAMES:
|
||||
raise ValueError("fixture input names/order differ")
|
||||
read_array(fixture, "x", (1, 1, 768))
|
||||
for name in ("cos", "sin"):
|
||||
read_array(fixture, name, (1, 1, 1, 64))
|
||||
read_array(fixture, "mask", (1, 1, 1, 257))
|
||||
for name in INPUT_NAMES[4:]:
|
||||
read_array(fixture, name, (1, 12, 256, 64))
|
||||
|
||||
comparisons = {}
|
||||
with np.load(args.cpu_reference, allow_pickle=False) as reference, np.load(
|
||||
args.device_output, allow_pickle=False
|
||||
) as output:
|
||||
if reference.files != OUTPUT_NAMES:
|
||||
raise ValueError("CPU output names/order differ")
|
||||
expected_keys = [f"output_{index}__0" for index in range(len(OUTPUT_NAMES))]
|
||||
if output.files != expected_keys:
|
||||
raise ValueError("device output names/order differ")
|
||||
for index, name in enumerate(OUTPUT_NAMES):
|
||||
shape = (1, 6562) if index == 0 else (1, 12, 1, 64)
|
||||
source_value = read_array(reference, name, shape)
|
||||
device_value = read_array(output, expected_keys[index], shape)
|
||||
comparisons[name] = compare(source_value, device_value)
|
||||
if name == "logits":
|
||||
comparisons[name]["cpu_top1"] = int(np.argmax(source_value[0]))
|
||||
comparisons[name]["device_top1"] = int(np.argmax(device_value[0]))
|
||||
comparisons[name]["top1_matches"] = bool(
|
||||
comparisons[name]["cpu_top1"] == comparisons[name]["device_top1"]
|
||||
)
|
||||
|
||||
report = {
|
||||
"scope": "one fixed-cache 20-layer MiniCPM-o speech-head decode step; no complete multimodal stream",
|
||||
"status": "component_inference_numeric_measured",
|
||||
"device": inferred["device"],
|
||||
"compile_job_id": compiled["job_id"],
|
||||
"inference_job_id": inferred["job_id"],
|
||||
"source_model_id": compiled["source_model_id"],
|
||||
"target_model_id": compiled["target_model_id"],
|
||||
"input_dataset_id": inferred["input_dataset_id"],
|
||||
"requested_run_options": inferred["options"],
|
||||
"files": files,
|
||||
"tensor_comparison": comparisons,
|
||||
"cache_max_relative_l2": max(comparisons[name]["relative_l2"] for name in OUTPUT_NAMES[1:]),
|
||||
"limits": [
|
||||
"The historical ONNX export does not attest the exact source checkpoint revision.",
|
||||
"One retained decode fixture cannot establish speech-token or generated-audio quality.",
|
||||
"Actual NPU placement requires a device profile; a request or QNN artifact alone is insufficient.",
|
||||
"Thinker, audio/vision encoders, vocoder, state loop and complete Omni request were not tested.",
|
||||
],
|
||||
}
|
||||
args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({
|
||||
"status": report["status"],
|
||||
"device": report["device"]["name"],
|
||||
"logits": comparisons["logits"],
|
||||
"cache_max_relative_l2": report["cache_max_relative_l2"],
|
||||
}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -59,17 +59,20 @@ def main() -> None:
|
||||
parser.add_argument("--inference-report", type=Path, required=True)
|
||||
parser.add_argument("--device-output", type=Path, required=True)
|
||||
parser.add_argument("--output-report", type=Path, required=True)
|
||||
parser.add_argument("--expected-compile-options", default=COMPILE_OPTIONS)
|
||||
parser.add_argument("--expected-run-options", default="--compute_unit npu")
|
||||
args = parser.parse_args()
|
||||
|
||||
compiled = json.loads(args.compile_report.read_text(encoding="utf-8"))
|
||||
inferred = json.loads(args.inference_report.read_text(encoding="utf-8"))
|
||||
if compiled.get("status") != "SUCCESS" or inferred.get("status") != "SUCCESS":
|
||||
raise ValueError("compile and inference must both be terminal successes")
|
||||
if compiled.get("source_model_id") != "mq3xre4rq" or compiled.get("options") != COMPILE_OPTIONS:
|
||||
if compiled.get("source_model_id") != "mq3xre4rq" or compiled.get("options") != args.expected_compile_options:
|
||||
raise ValueError("compile source or options differ from the pinned talker artifact")
|
||||
if inferred.get("model_id") != compiled.get("target_model_id"):
|
||||
raise ValueError("inference did not use the audited target artifact")
|
||||
if inferred.get("input_dataset_id") != "d7dw6pgy2" or inferred.get("options") != "--compute_unit npu":
|
||||
if (inferred.get("input_dataset_id") != "d7dw6pgy2"
|
||||
or inferred.get("options") != args.expected_run_options):
|
||||
raise ValueError("inference fixture or requested compute unit differs")
|
||||
if inferred.get("device") != compiled.get("device"):
|
||||
raise ValueError("compile/inference device metadata differs")
|
||||
@@ -140,7 +143,7 @@ def main() -> None:
|
||||
"limits": [
|
||||
"The historical source export does not attest an exact checkpoint revision.",
|
||||
"The fixture is synthetic and has no token-to-audio quality tolerance.",
|
||||
"Requested NPU is not actual placement without a device profile.",
|
||||
"Requested compute unit is not actual placement without a device profile.",
|
||||
"No predictor, vocoder, persistent cache loop or complete TTS stream was tested.",
|
||||
],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure policy-action sensitivity to a hosted Cosmos encoder latent.
|
||||
|
||||
The full policy runs on local CPU with identical synthetic state/noise. Its
|
||||
encoder is replaced by pinned source/device latent tensors from one fixture.
|
||||
This is a numerical handoff probe, not a device-local or robot-task run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
CHECKPOINT_SHA256 = "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313"
|
||||
FIXTURE_SHA256 = "dcbaf1391ca914d4bbccc1cb384daf05666e771d9cdae90c4f069ec8fcf05368"
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
with path.open("rb") as source:
|
||||
return hashlib.file_digest(source, "sha256").hexdigest()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for name in ("model-dir", "cosmos-dir", "processor-dir", "fixture", "device-output", "audit-report", "output"):
|
||||
parser.add_argument(f"--{name}", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
os.environ["INTERNVLA_A1_COSMOS_DIR"] = str(args.cosmos_dir.resolve(strict=True))
|
||||
os.environ["INTERNVLA_A1_PROCESSOR_DIR"] = str(args.processor_dir.resolve(strict=True))
|
||||
os.environ["HF_HUB_OFFLINE"] = "1"
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.registry import initialize_model
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
checkpoint = args.model_dir / "model.safetensors"
|
||||
if sha256(checkpoint) != CHECKPOINT_SHA256 or sha256(args.fixture) != FIXTURE_SHA256:
|
||||
raise ValueError("checkpoint or encoder fixture differs from pinned source")
|
||||
audit = json.loads(args.audit_report.read_text(encoding="utf-8"))
|
||||
if (audit.get("status") != "component_inference_numeric_measured"
|
||||
or audit.get("device", {}).get("name") != "Samsung Galaxy S25"
|
||||
or audit.get("files", {}).get("device_output", {}).get("sha256") != sha256(args.device_output)):
|
||||
raise ValueError("hosted latent lacks audited S25 provenance")
|
||||
with np.load(args.fixture, allow_pickle=False) as fixture:
|
||||
source_latent = np.asarray(fixture["pattern_reference"])
|
||||
with np.load(args.device_output, allow_pickle=False) as output:
|
||||
if output.files != ["output_0__0"]:
|
||||
raise ValueError("unexpected hosted output layout")
|
||||
device_latent = np.asarray(output["output_0__0"])
|
||||
if (source_latent.shape != (6, 16, 32, 32) or device_latent.shape != source_latent.shape
|
||||
or source_latent.dtype != np.float32 or device_latent.dtype != np.float32
|
||||
or not np.isfinite(source_latent).all() or not np.isfinite(device_latent).all()):
|
||||
raise ValueError("latents violate the finite FP32 six-frame contract")
|
||||
|
||||
config = OmniDiffusionConfig(
|
||||
model=str(args.model_dir.resolve(strict=True)),
|
||||
model_class_name="InternVLAA1Pipeline",
|
||||
dtype=torch.bfloat16,
|
||||
custom_pipeline_args={
|
||||
"device": "cpu", "dtype": "bfloat16", "compile_model": False,
|
||||
"enable_regional_compile": False, "enable_warmup": False,
|
||||
"strict_load": True,
|
||||
"processor_model_name": str(args.processor_dir.resolve(strict=True)),
|
||||
},
|
||||
)
|
||||
started = time.perf_counter()
|
||||
pipeline = initialize_model(config)
|
||||
load_seconds = time.perf_counter() - started
|
||||
if pipeline.runtime_mode() != "real_checkpoint_loaded":
|
||||
raise RuntimeError("real policy checkpoint was not loaded")
|
||||
if {parameter.device.type for parameter in pipeline.policy.parameters()} != {"cpu"}:
|
||||
raise RuntimeError("policy parameters did not stay on CPU")
|
||||
inputs = pipeline._build_fake_batch_inputs()
|
||||
noise = torch.zeros((1, pipeline.config.chunk_size, pipeline.config.max_action_dim), dtype=torch.float32)
|
||||
cosmos = pipeline.policy.model.cosmos
|
||||
original_encoder = cosmos._enc_model
|
||||
|
||||
class PinnedLatent(torch.nn.Module):
|
||||
def __init__(self, latent: np.ndarray) -> None:
|
||||
super().__init__()
|
||||
self.latent = torch.from_numpy(latent.copy())
|
||||
self.calls = 0
|
||||
|
||||
def forward(self, pixels: torch.Tensor) -> torch.Tensor:
|
||||
if tuple(pixels.shape) != (6, 3, 256, 256) or pixels.device.type != "cpu":
|
||||
raise RuntimeError("unexpected policy encoder input")
|
||||
self.calls += 1
|
||||
return self.latent.to(dtype=pixels.dtype)
|
||||
|
||||
def actions(encoder: PinnedLatent) -> np.ndarray:
|
||||
cosmos._enc_model = encoder
|
||||
result = pipeline.forward(DiffusionRequestBatch(requests=[
|
||||
OmniDiffusionRequest(
|
||||
prompt="",
|
||||
sampling_params=OmniDiffusionSamplingParams(extra_args={
|
||||
"batch_inputs": inputs, "noise": noise, "decode_image": False,
|
||||
}),
|
||||
request_id="internvla-hosted-latent-synthetic",
|
||||
)
|
||||
]))
|
||||
if result.error:
|
||||
raise RuntimeError(result.error)
|
||||
tensor = result.output["payload"]["actions"]
|
||||
if tuple(tensor.shape) != (1, 50, 32) or tensor.device.type != "cpu" or not torch.isfinite(tensor).all():
|
||||
raise RuntimeError("bad policy action tensor")
|
||||
return tensor.detach().float().cpu().contiguous().numpy()
|
||||
|
||||
source_encoder = PinnedLatent(source_latent)
|
||||
hosted_encoder = PinnedLatent(device_latent)
|
||||
try:
|
||||
source_actions = actions(source_encoder)
|
||||
hosted_actions = actions(hosted_encoder)
|
||||
source_repeat = actions(source_encoder)
|
||||
finally:
|
||||
cosmos._enc_model = original_encoder
|
||||
if not np.array_equal(source_actions, source_repeat):
|
||||
raise RuntimeError("CPU source policy changed across the paired calls")
|
||||
source = source_actions.astype(np.float64).ravel()
|
||||
hosted = hosted_actions.astype(np.float64).ravel()
|
||||
delta = source - hosted
|
||||
report = {
|
||||
"scope": "S25 hosted Cosmos latent injected into local CPU InternVLA policy with synthetic state/noise; no device-local policy or robot-task result",
|
||||
"status": "synthetic_action_sensitivity_measured",
|
||||
"policy_checkpoint_sha256": CHECKPOINT_SHA256,
|
||||
"fixture_sha256": FIXTURE_SHA256,
|
||||
"hosted_output_sha256": sha256(args.device_output),
|
||||
"hosted_inference_job_id": audit["inference_job_id"],
|
||||
"policy_torch": torch.__version__,
|
||||
"policy_load_seconds_not_request_timing": load_seconds,
|
||||
"encoder_calls": {"source": source_encoder.calls, "hosted": hosted_encoder.calls},
|
||||
"action_shape": list(source_actions.shape),
|
||||
"source_repeat_bitwise_equal": True,
|
||||
"action_relative_l2": float(np.linalg.norm(delta) / np.linalg.norm(source)),
|
||||
"action_max_abs": float(np.max(np.abs(delta))),
|
||||
"action_cosine": float(np.dot(source, hosted) / (np.linalg.norm(source) * np.linalg.norm(hosted))),
|
||||
"source_actions_sha256": hashlib.sha256(source_actions.tobytes()).hexdigest(),
|
||||
"hosted_actions_sha256": hashlib.sha256(hosted_actions.tobytes()).hexdigest(),
|
||||
"limits": [
|
||||
"The policy ran on local CPU; the device only executed the isolated encoder fixture.",
|
||||
"The encoder input was a synthetic pattern and policy state/noise were synthetic zeros.",
|
||||
"No real-observation reference actions, physical units/time, or robot-task tolerance exist.",
|
||||
"No latency, memory admission, cancellation or sustained device power gate was tested.",
|
||||
],
|
||||
}
|
||||
args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({key: report[key] for key in (
|
||||
"status", "action_relative_l2", "action_max_abs", "action_cosine", "source_repeat_bitwise_equal"
|
||||
)}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,9 +1,11 @@
|
||||
# 2026-09-24 Qualcomm component expansion
|
||||
|
||||
The [Qwen3-TTS vocoder study](evidence/qwen_tts_vocoder_qualcomm/README.md) reuses a retained fixed-shape Code2Wav export and historical Workbench input to test numerical behavior on SA8775P and other Qualcomm settings. The fresh SA8775P GPU-requested inference produces a saturated waveform, while the same binary's CPU-requested output on the exact SA device matches ONNX CPU within 4.13e-6 relative L2. RB3 produces an unsaturated but 15.73%-divergent waveform; its fresh 100-sample mixed CPU/GPU component p50/p95 was 3.099/3.133 s, not a usable TTS timing claim. X Elite ONNX CPU output matches the local ONNX Runtime reference on the retained fixture; its 94-sample component p50/p95 was 6.384/6.644 s with all execution-detail rows on CPU. Corrected X Elite DirectML inference and profile both failed during provider initialization; separate FP16 QNN DLC compilation passed, with NPU-requested inference pending. SA CPU placement profiling and FP32-preserving GPU control remain pending. No complete on-device TTS stream or model-quality qualification is claimed.
|
||||
The [Qwen3-TTS vocoder study](evidence/qwen_tts_vocoder_qualcomm/README.md) reuses a retained fixed-shape Code2Wav export and historical Workbench input to test numerical behavior on SA8775P and other Qualcomm settings. The fresh SA8775P GPU-requested inference produces a saturated waveform, while the same binary's CPU-requested output on the exact SA device matches ONNX CPU within 4.13e-6 relative L2. A FP32-preserving GPU-option control still saturated 84.89% of samples, so that option did not repair the tested path. RB3 produces an unsaturated but 15.73%-divergent waveform; its fresh 100-sample mixed CPU/GPU component p50/p95 was 3.099/3.133 s, not a usable TTS timing claim. X Elite ONNX CPU output matches the local ONNX Runtime reference on the retained fixture; its 94-sample component p50/p95 was 6.384/6.644 s with all execution-detail rows on CPU. Corrected X Elite DirectML inference and profile both failed during provider initialization; separate FP16 QNN DLC compilation passed, with NPU-requested inference pending. SA CPU placement profiling remains pending. No complete on-device TTS stream or model-quality qualification is claimed.
|
||||
|
||||
A local HX370 x86-64 LiteRT CPU control of the identical SA/RB3 TFLite binary matched the original ONNX CPU waveform within 3.45e-6 relative L2. The SA saturation is therefore not universal to the converted artifact; hosted CPU and GPU precision controls are still needed to isolate its device/runtime cause.
|
||||
|
||||
The [talker decode-step attempt](evidence/qwen_tts_talker_qualcomm/README.md) pins a separate 28-layer source and synthetic cache fixture. Historical FP16 QNN context-binary compiles passed for S24, S25, SA8775P and X Elite, with same-fixture inference pending; RB3 rejected that floating-point-input artifact. Compile success is not a numerical or end-to-end pass.
|
||||
The [talker decode-step attempt](evidence/qwen_tts_talker_qualcomm/README.md) pins a separate 28-layer source and synthetic cache fixture. Historical FP16 QNN context-binary inference executed on S24, S25, SA8775P and X Elite, but all four changed the CPU top-1 speech token with 8.57–9.43% logits relative L2. These artifacts are numerically unqualified on the fixture. RB3 rejected the floating-point-input artifact, and an X Elite ONNX CPU fallback compile is pending. No complete talker loop or TTS stream passed.
|
||||
|
||||
The [InternVLA Cosmos S25 attempt](evidence/internvla_cosmos_qualcomm/README.md) repackages the real policy's image encoder from its parity-tested ONNX export. FLOAT16 QNN DLC compilation passed; same-fixture NPU-requested inference was pending. Full action-policy support remains unqualified.
|
||||
The [MiniCPM-o speech-head study](evidence/minicpmo_tts_qualcomm/README.md) recovered the real 20-layer ONNX source and a matching retained fixture. S24 and S25 FP16 context-binary outputs preserved the ONNX CPU top-1 speech token with 0.346%/0.360% logits relative L2 on one step. FP16 QNN DLC compiles passed for SA8775P, X Elite and RB3, with same-fixture NPU-requested inference pending. The complete multimodal model remains unqualified on those devices.
|
||||
|
||||
The [InternVLA Cosmos S25 attempt](evidence/internvla_cosmos_qualcomm/README.md) repackages the real policy's image encoder from its parity-tested ONNX export. FLOAT16 QNN DLC inference returned a latent within 0.0524% relative L2 of the source on one pattern. Injecting it into the real policy on local CPU changed synthetic actions by 0.742% relative L2, without a task tolerance. NPU placement profiling was pending; device-local action-policy support remains unqualified.
|
||||
+3
-3
@@ -2,8 +2,8 @@
|
||||
|
||||
This tests the **real Place_Markpen Cosmos CI8x8 image encoder component**, not the action policy or complete InternVLA. The [source export](../../../e2e_expansion_20260923/evidence/internvla_cosmos_amd_npu/README.md) pins the checkpoint, FP32 `[6,3,256,256]` input, `[6,16,32,32]` output, two CPU parity fixtures and the original external-data ONNX files. The external data were embedded into one 125,112,088-byte ONNX file outside Git at `/home/zhout/project/edge_infer/models/InternVLA-cosmos-onnx/cosmos_encoder_embedded.onnx` (SHA256 `656a5263f584851a1457ef6dcf60b995f7c67b056b3c90a4f33c814f7caf9ec9`). The embedded form retained reference-relative ONNX CPU L2 errors of `8.74e-7` and `2.29e-6` on ramp and pattern fixtures.
|
||||
|
||||
Workbench source model `mn4okj3rq` was [submitted for Galaxy S25 QNN DLC compilation](s25/compile_submission.json) as job `jpxl4d1jp`, with `default_graph_htp_precision=FLOAT16`. The [compile](s25/compile_report.json) succeeded and produced target model `mqkp12rxm`. A [same-fixture pattern inference](s25/inference_submission.json) requesting NPU was submitted as `jp8ex43zp` and remained pending at this evidence snapshot. Compile success does not establish NPU execution, output parity, action quality, latency, or full-policy support.
|
||||
Workbench source model `mn4okj3rq` was [submitted for Galaxy S25 QNN DLC compilation](s25/compile_submission.json) as job `jpxl4d1jp`, with `default_graph_htp_precision=FLOAT16`. The [compile](s25/compile_report.json) succeeded and produced target model `mqkp12rxm`. [Same-fixture pattern inference](s25/inference_report.json) requesting NPU also succeeded. The [audit](s25/audit_report.json) found **0.0524% relative L2 / 65.61 dB SNR** versus the pinned source latent, with finite FP32 `[6,16,32,32]` output. This establishes numerical behavior on one synthetic encoder fixture; requested NPU still needs the [placement profile](s25/profile_submission.json), which was pending at this snapshot.
|
||||
|
||||
The [generic collector](../../../../experiments/collect_aihub_component.py) can retrieve the pending output, and the [Cosmos auditor](../../../../experiments/audit_internvla_cosmos_qualcomm.py) checks source/fixture hashes, target/device/dataset identity, latent shape and finite numerical difference against the pinned source reference. Neither script converts an encoder-only result to policy support.
|
||||
The [generic collector](../../../../experiments/collect_aihub_component.py) preserves terminal jobs, and the [Cosmos auditor](../../../../experiments/audit_internvla_cosmos_qualcomm.py) checks source/fixture hashes, target/device/dataset identity, latent shape and finite numerical difference against the pinned source reference. A separate [synthetic action sensitivity check](s25/action_sensitivity.json) injected either the pinned source or S25 latent into the same real Place_Markpen policy on **local CPU**, with zero state/noise. The [reproducer](../../../../experiments/probe_internvla_hosted_latent_actions.py) returned finite `[1,50,32]` actions; hosted-vs-source action relative L2 was **0.742%**, maximum absolute difference **0.0115**, cosine **0.999973**. Repeating the source policy gave bitwise identical actions. This is a cross-device numerical handoff experiment, not a device-local policy or robot-task pass; no real-observation action tolerance is known.
|
||||
|
||||
Next: retrieve and audit the device latent, inspect actual NPU placement with a profile if numerical error is acceptable, then test action-policy impact and a device-local stateful policy. Full multimodal observation processing, admission, cancellation and sustained profiling remain open.
|
||||
Next: inspect actual NPU placement with the pending profile, test representative real observations and action tolerances, then build and validate a device-local stateful policy. Full multimodal observation processing, admission, cancellation and sustained profiling remain open.
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"scope": "S25 hosted Cosmos latent injected into local CPU InternVLA policy with synthetic state/noise; no device-local policy or robot-task result",
|
||||
"status": "synthetic_action_sensitivity_measured",
|
||||
"policy_checkpoint_sha256": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313",
|
||||
"fixture_sha256": "dcbaf1391ca914d4bbccc1cb384daf05666e771d9cdae90c4f069ec8fcf05368",
|
||||
"hosted_output_sha256": "69647f9fe7207dacc054330c7174d08bdcb4ecf8611f91efc1eb70065418b389",
|
||||
"hosted_inference_job_id": "jp8ex43zp",
|
||||
"policy_torch": "2.13.0+cpu",
|
||||
"policy_load_seconds_not_request_timing": 34.37513727898477,
|
||||
"encoder_calls": {
|
||||
"source": 2,
|
||||
"hosted": 1
|
||||
},
|
||||
"action_shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"source_repeat_bitwise_equal": true,
|
||||
"action_relative_l2": 0.007421938467116418,
|
||||
"action_max_abs": 0.01148313283920288,
|
||||
"action_cosine": 0.9999725460323058,
|
||||
"source_actions_sha256": "d20b4804ce549c25e46590f4057d5d9eceb9e331c7015e3fc90cf79a234bf018",
|
||||
"hosted_actions_sha256": "cd1f1a1b8883c74e36a553b6a6d98631beaec22990b70b5114f1e35599e2ec6d",
|
||||
"limits": [
|
||||
"The policy ran on local CPU; the device only executed the isolated encoder fixture.",
|
||||
"The encoder input was a synthetic pattern and policy state/noise were synthetic zeros.",
|
||||
"No real-observation reference actions, physical units/time, or robot-task tolerance exist.",
|
||||
"No latency, memory admission, cancellation or sustained device power gate was tested."
|
||||
]
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"scope": "one six-frame Cosmos image encoder on synthetic pattern; no complete InternVLA policy",
|
||||
"status": "component_inference_numeric_measured",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S25",
|
||||
"os": "15",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8-elite-for-galaxy",
|
||||
"chipset:sm8750-ac",
|
||||
"hexagon:v79",
|
||||
"soc-model:69",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"source_model_id": "mn4okj3rq",
|
||||
"target_model_id": "mqkp12rxm",
|
||||
"compile_job_id": "jpxl4d1jp",
|
||||
"inference_job_id": "jp8ex43zp",
|
||||
"input_dataset_id": "d26qo8p57",
|
||||
"requested_run_options": "--compute_unit npu",
|
||||
"files": {
|
||||
"source_onnx": {
|
||||
"bytes": 125112088,
|
||||
"sha256": "656a5263f584851a1457ef6dcf60b995f7c67b056b3c90a4f33c814f7caf9ec9"
|
||||
},
|
||||
"fixture": {
|
||||
"bytes": 2081321,
|
||||
"sha256": "dcbaf1391ca914d4bbccc1cb384daf05666e771d9cdae90c4f069ec8fcf05368"
|
||||
},
|
||||
"device_output": {
|
||||
"bytes": 185871,
|
||||
"sha256": "69647f9fe7207dacc054330c7174d08bdcb4ecf8611f91efc1eb70065418b389"
|
||||
}
|
||||
},
|
||||
"output_dtype": "float32",
|
||||
"source_vs_device": {
|
||||
"relative_l2": 0.0005243597769688565,
|
||||
"snr_db": 65.60741259861898,
|
||||
"max_abs_error": 0.007274627685546875,
|
||||
"cosine": 0.999999865135908
|
||||
},
|
||||
"limits": [
|
||||
"The fixture is synthetic; no real-observation or action-quality tolerance is established.",
|
||||
"Requested NPU is not actual per-node placement without a device profile.",
|
||||
"The action policy, state lifecycle, admission and complete request were not run on this device."
|
||||
]
|
||||
}
|
||||
BIN
Binary file not shown.
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:48:37.390932+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "jp8ex43zp",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jp8ex43zp/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"options": "--compute_unit npu",
|
||||
"hub_version": "aihub-2026.09.11.0",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S25",
|
||||
"os": "15",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8-elite-for-galaxy",
|
||||
"chipset:sm8750-ac",
|
||||
"hexagon:v79",
|
||||
"soc-model:69",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"model_id": "mqkp12rxm",
|
||||
"input_dataset_id": "d26qo8p57",
|
||||
"outputs": "device_output.npz"
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:51:08.958776+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "jpxl4d1jp",
|
||||
"target_model_id": "mqkp12rxm",
|
||||
"device": "Samsung Galaxy S25",
|
||||
"options": "--compute_unit npu",
|
||||
"kind": "profile",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jp1njvm8g",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jp1njvm8g/"
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
# MiniCPM-o 4.5 speech-head decode step across Qualcomm devices
|
||||
|
||||
This is a **single fixed-cache 20-layer speech-head step**, not the complete thinker, audio/vision encoders, vocoder or streaming MiniCPM-o pipeline. The retained Workbench source ONNX model `mn4gjjpvn` was recovered outside Git as `/home/zhout/project/edge_infer/models/minicpmo45_tts_step_L256_20260924.onnx` (776,341,465 bytes, SHA256 `78cf64804f11ad269ee4180da61588ccc5eefe2942773227345499756e4cc229`). It has FP32 hidden input `[1,1,768]`, rotary/mask inputs, 20 K/V cache pairs at length 256, and 6,562 logits plus 40 new-cache outputs. The historical export does not attest its exact checkpoint revision.
|
||||
|
||||
The [S25 historical inference](s25/inference_report.json) used dataset `d7gwxe3y2`; the [S24 historical inference](s24/inference_report.json) used `d7zn8rk57`. Downloaded inputs from both dataset IDs produced the same 44-array `.npz` SHA256 `65919ea1f351feec33c1415a3e3db76d60bc9c2beb3d49a72af75dcb418e83b4`. The retained fixture is outside Git under `/home/zhout/project/edge_infer/models/minicpmo45_tts_step_fixture_s25_20260924.npz`. ONNX Runtime 1.29 CPU produced a 41-output reference (SHA256 `902adcceadbe761d938ef2ade51cf953c8917ea1abfaee33a10c84e300c1bd96`), also outside Git.
|
||||
|
||||
The historical [S24](s24/compile_report.json) and [S25](s25/compile_report.json) FP16 QNN context-binary compiles and inference jobs succeeded. Against the **same original ONNX CPU fixture**, [S24](s24/audit_report.json) logits relative L2 was **0.3459%** and max K/V relative L2 **1.235%**; [S25](s25/audit_report.json) logits relative L2 was **0.3605%** and max K/V relative L2 **1.311%**. Both preserved the CPU top-1 speech token **1867** on this one synthetic step. The raw [S24](s24/device_output.npz) and [S25](s25/device_output.npz) arrays are retained. These values are numerical component evidence, without a generated-audio quality threshold or actual NPU placement profile.
|
||||
|
||||
New FP16 QNN DLC [SA8775P](sa8775p/compile_report.json), [Snapdragon X Elite](xelite/compile_report.json) and [RB3 Gen 2](rb3/compile_report.json) compiles passed from the same source. Same-fixture NPU-requested inference jobs were [submitted for SA8775P](sa8775p/inference_submission.json), [X Elite](xelite/inference_submission.json) and [RB3](rb3/inference_submission.json), but were pending at this snapshot. A compile cannot establish device execution or numerical parity. The [collector](../../../../experiments/collect_aihub_component.py) retrieves terminal jobs, and the [auditor](../../../../experiments/audit_minicpmo_tts_qualcomm.py) pins hashes and checks all 41 output tensors.
|
||||
|
||||
Next: audit those device outputs and placement, then qualify the entire model with real multimodal inputs, stateful speech generation, task/audio quality, memory admission, cancellation and sustained power/thermal behavior. AI Hub is the test facility, not a deployment dependency.
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"retrieved_utc": "2026-09-23T23:42:08.019012+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"job_id": "j5wl71l6p",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/j5wl71l6p/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"source_model_id": "mn4gjjpvn",
|
||||
"target_model_id": "mnwv5rj3q",
|
||||
"device": {
|
||||
"name": "Dragonwing RB3 Gen 2 Vision Kit",
|
||||
"os": "1.6",
|
||||
"attributes": [
|
||||
"os:qc_linux",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-oe-linux-gcc11",
|
||||
"format:iot",
|
||||
"chipset:qualcomm-qcs6490",
|
||||
"chipset:qcs6490",
|
||||
"chipset:qualcomm-dragonwing-qcs6490",
|
||||
"hexagon:v68",
|
||||
"soc-model:93",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"options": "--target_runtime qnn_dlc --qnn_options default_graph_htp_precision=FLOAT16",
|
||||
"hub_version": "aihub-2026.09.11.0"
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:38:27.062001+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_model_id": "mn4gjjpvn",
|
||||
"source_sha256": "78cf64804f11ad269ee4180da61588ccc5eefe2942773227345499756e4cc229",
|
||||
"device": "Dragonwing RB3 Gen 2 Vision Kit",
|
||||
"options": "--target_runtime qnn_dlc --qnn_options default_graph_htp_precision=FLOAT16",
|
||||
"kind": "compile",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "j5wl71l6p",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/j5wl71l6p/"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:42:09.154844+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "j5wl71l6p",
|
||||
"target_model_id": "mnwv5rj3q",
|
||||
"input_dataset_id": "d7gwxe3y2",
|
||||
"device": "Dragonwing RB3 Gen 2 Vision Kit",
|
||||
"options": "--compute_unit npu",
|
||||
"kind": "inference",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jpvld92j5",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jpvld92j5/"
|
||||
}
|
||||
+305
@@ -0,0 +1,305 @@
|
||||
{
|
||||
"scope": "one fixed-cache 20-layer MiniCPM-o speech-head decode step; no complete multimodal stream",
|
||||
"status": "component_inference_numeric_measured",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S24",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8gen3",
|
||||
"chipset:sm8650",
|
||||
"hexagon:v75",
|
||||
"soc-model:57",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"compile_job_id": "jp2ryv3qg",
|
||||
"inference_job_id": "jp0m0646g",
|
||||
"source_model_id": "mn4gjjpvn",
|
||||
"target_model_id": "mqp411wgq",
|
||||
"input_dataset_id": "d7zn8rk57",
|
||||
"requested_run_options": "",
|
||||
"files": {
|
||||
"source_onnx": {
|
||||
"bytes": 776341465,
|
||||
"sha256": "78cf64804f11ad269ee4180da61588ccc5eefe2942773227345499756e4cc229"
|
||||
},
|
||||
"fixture": {
|
||||
"bytes": 29247541,
|
||||
"sha256": "65919ea1f351feec33c1415a3e3db76d60bc9c2beb3d49a72af75dcb418e83b4"
|
||||
},
|
||||
"cpu_reference": {
|
||||
"bytes": 148471,
|
||||
"sha256": "902adcceadbe761d938ef2ade51cf953c8917ea1abfaee33a10c84e300c1bd96"
|
||||
},
|
||||
"device_output": {
|
||||
"bytes": 105697,
|
||||
"sha256": "e4e9818b0eb2edb48d569f64de5fe80a79b9b49a89ab6983425d1b33cf89f3bb"
|
||||
}
|
||||
},
|
||||
"tensor_comparison": {
|
||||
"logits": {
|
||||
"relative_l2": 0.0034590011716080673,
|
||||
"max_abs_error": 0.01756751537322998,
|
||||
"source_rms": 1.4974441755558239,
|
||||
"device_rms": 1.4976054665256364,
|
||||
"cpu_top1": 1867,
|
||||
"device_top1": 1867,
|
||||
"top1_matches": true
|
||||
},
|
||||
"k_new_0": {
|
||||
"relative_l2": 0.0004728083634232632,
|
||||
"max_abs_error": 0.003048419952392578,
|
||||
"source_rms": 1.307108108274551,
|
||||
"device_rms": 1.3070441579437349
|
||||
},
|
||||
"v_new_0": {
|
||||
"relative_l2": 0.0003447410805552075,
|
||||
"max_abs_error": 0.0008349418640136719,
|
||||
"source_rms": 0.5083994128636046,
|
||||
"device_rms": 0.5083716190314295
|
||||
},
|
||||
"k_new_1": {
|
||||
"relative_l2": 0.0005754907265917865,
|
||||
"max_abs_error": 0.002339482307434082,
|
||||
"source_rms": 0.7542700462780494,
|
||||
"device_rms": 0.7542548389751653
|
||||
},
|
||||
"v_new_1": {
|
||||
"relative_l2": 0.0009302397915078979,
|
||||
"max_abs_error": 0.0002478659152984619,
|
||||
"source_rms": 0.0662130257652199,
|
||||
"device_rms": 0.06621815014592335
|
||||
},
|
||||
"k_new_2": {
|
||||
"relative_l2": 0.0012956163895706514,
|
||||
"max_abs_error": 0.0056171417236328125,
|
||||
"source_rms": 0.9817805619626084,
|
||||
"device_rms": 0.9819350618974992
|
||||
},
|
||||
"v_new_2": {
|
||||
"relative_l2": 0.001349323495030849,
|
||||
"max_abs_error": 0.001597076654434204,
|
||||
"source_rms": 0.24438175712240315,
|
||||
"device_rms": 0.24441806437946306
|
||||
},
|
||||
"k_new_3": {
|
||||
"relative_l2": 0.004157877140839533,
|
||||
"max_abs_error": 0.013773024082183838,
|
||||
"source_rms": 0.9312003114105484,
|
||||
"device_rms": 0.9324908359787388
|
||||
},
|
||||
"v_new_3": {
|
||||
"relative_l2": 0.0044989514889889675,
|
||||
"max_abs_error": 0.006326019763946533,
|
||||
"source_rms": 0.27179250329538757,
|
||||
"device_rms": 0.2722213544309874
|
||||
},
|
||||
"k_new_4": {
|
||||
"relative_l2": 0.007864916658441525,
|
||||
"max_abs_error": 0.039246320724487305,
|
||||
"source_rms": 1.2078244698126999,
|
||||
"device_rms": 1.21157035045722
|
||||
},
|
||||
"v_new_4": {
|
||||
"relative_l2": 0.00850423216372736,
|
||||
"max_abs_error": 0.01918625831604004,
|
||||
"source_rms": 0.4929021962159226,
|
||||
"device_rms": 0.49434128109868714
|
||||
},
|
||||
"k_new_5": {
|
||||
"relative_l2": 0.007755617989875097,
|
||||
"max_abs_error": 0.04616549611091614,
|
||||
"source_rms": 1.0960093751879882,
|
||||
"device_rms": 1.0970680463473175
|
||||
},
|
||||
"v_new_5": {
|
||||
"relative_l2": 0.012347926009067344,
|
||||
"max_abs_error": 0.028903603553771973,
|
||||
"source_rms": 0.49114616760529134,
|
||||
"device_rms": 0.48969865397772927
|
||||
},
|
||||
"k_new_6": {
|
||||
"relative_l2": 0.006333181458689364,
|
||||
"max_abs_error": 0.03584308922290802,
|
||||
"source_rms": 1.1726536779527166,
|
||||
"device_rms": 1.1734537524365503
|
||||
},
|
||||
"v_new_6": {
|
||||
"relative_l2": 0.009530458560541577,
|
||||
"max_abs_error": 0.016398705542087555,
|
||||
"source_rms": 0.5369044199454948,
|
||||
"device_rms": 0.5364217595250447
|
||||
},
|
||||
"k_new_7": {
|
||||
"relative_l2": 0.004093385306149082,
|
||||
"max_abs_error": 0.024735689163208008,
|
||||
"source_rms": 1.6917150856043028,
|
||||
"device_rms": 1.6942228360537472
|
||||
},
|
||||
"v_new_7": {
|
||||
"relative_l2": 0.007156864648832234,
|
||||
"max_abs_error": 0.01977591961622238,
|
||||
"source_rms": 0.8116460286615261,
|
||||
"device_rms": 0.8109094659868129
|
||||
},
|
||||
"k_new_8": {
|
||||
"relative_l2": 0.005078716324257598,
|
||||
"max_abs_error": 0.0369257926940918,
|
||||
"source_rms": 1.5021910590457033,
|
||||
"device_rms": 1.5033731557981107
|
||||
},
|
||||
"v_new_8": {
|
||||
"relative_l2": 0.00699838877358874,
|
||||
"max_abs_error": 0.030480504035949707,
|
||||
"source_rms": 0.8807572129778053,
|
||||
"device_rms": 0.8792153267665787
|
||||
},
|
||||
"k_new_9": {
|
||||
"relative_l2": 0.004985529541790878,
|
||||
"max_abs_error": 0.01878225803375244,
|
||||
"source_rms": 1.1054146201792987,
|
||||
"device_rms": 1.1057376686425842
|
||||
},
|
||||
"v_new_9": {
|
||||
"relative_l2": 0.00801596002266864,
|
||||
"max_abs_error": 0.018241047859191895,
|
||||
"source_rms": 0.6580213796976341,
|
||||
"device_rms": 0.656717882498813
|
||||
},
|
||||
"k_new_10": {
|
||||
"relative_l2": 0.004657784024943821,
|
||||
"max_abs_error": 0.0236513614654541,
|
||||
"source_rms": 1.325937063058518,
|
||||
"device_rms": 1.326826314603149
|
||||
},
|
||||
"v_new_10": {
|
||||
"relative_l2": 0.006978349276760429,
|
||||
"max_abs_error": 0.022747039794921875,
|
||||
"source_rms": 0.848351970479369,
|
||||
"device_rms": 0.8469693150737657
|
||||
},
|
||||
"k_new_11": {
|
||||
"relative_l2": 0.0035862470520054046,
|
||||
"max_abs_error": 0.008336663246154785,
|
||||
"source_rms": 0.6808336872062942,
|
||||
"device_rms": 0.6804267801348977
|
||||
},
|
||||
"v_new_11": {
|
||||
"relative_l2": 0.006431017556092984,
|
||||
"max_abs_error": 0.008693762123584747,
|
||||
"source_rms": 0.3676426367961426,
|
||||
"device_rms": 0.36708839120124764
|
||||
},
|
||||
"k_new_12": {
|
||||
"relative_l2": 0.004340480587337765,
|
||||
"max_abs_error": 0.009877800941467285,
|
||||
"source_rms": 0.6128882051701765,
|
||||
"device_rms": 0.6122587094609966
|
||||
},
|
||||
"v_new_12": {
|
||||
"relative_l2": 0.0060537699679929285,
|
||||
"max_abs_error": 0.007887393236160278,
|
||||
"source_rms": 0.34029547623567724,
|
||||
"device_rms": 0.3398734072125847
|
||||
},
|
||||
"k_new_13": {
|
||||
"relative_l2": 0.004308169820748388,
|
||||
"max_abs_error": 0.008440196514129639,
|
||||
"source_rms": 0.613775761616868,
|
||||
"device_rms": 0.6134385095929898
|
||||
},
|
||||
"v_new_13": {
|
||||
"relative_l2": 0.006703524176478603,
|
||||
"max_abs_error": 0.008816160261631012,
|
||||
"source_rms": 0.36907442918730654,
|
||||
"device_rms": 0.3684204833095106
|
||||
},
|
||||
"k_new_14": {
|
||||
"relative_l2": 0.004225057190205451,
|
||||
"max_abs_error": 0.010796666145324707,
|
||||
"source_rms": 0.5980841827735426,
|
||||
"device_rms": 0.5976820809371475
|
||||
},
|
||||
"v_new_14": {
|
||||
"relative_l2": 0.0062976108818995746,
|
||||
"max_abs_error": 0.006353095173835754,
|
||||
"source_rms": 0.3506656009170703,
|
||||
"device_rms": 0.3501709716441067
|
||||
},
|
||||
"k_new_15": {
|
||||
"relative_l2": 0.0038634610570975423,
|
||||
"max_abs_error": 0.008877158164978027,
|
||||
"source_rms": 0.622821559168189,
|
||||
"device_rms": 0.6222562783246591
|
||||
},
|
||||
"v_new_15": {
|
||||
"relative_l2": 0.005683245988946143,
|
||||
"max_abs_error": 0.007857635617256165,
|
||||
"source_rms": 0.41823950764909973,
|
||||
"device_rms": 0.41772393100381044
|
||||
},
|
||||
"k_new_16": {
|
||||
"relative_l2": 0.0033562433354865978,
|
||||
"max_abs_error": 0.007819771766662598,
|
||||
"source_rms": 0.6754924715823191,
|
||||
"device_rms": 0.6752249152520644
|
||||
},
|
||||
"v_new_16": {
|
||||
"relative_l2": 0.005971706910746076,
|
||||
"max_abs_error": 0.010938256978988647,
|
||||
"source_rms": 0.4659773620123202,
|
||||
"device_rms": 0.46548628110240814
|
||||
},
|
||||
"k_new_17": {
|
||||
"relative_l2": 0.0035642247996178424,
|
||||
"max_abs_error": 0.008532702922821045,
|
||||
"source_rms": 0.6354120615871959,
|
||||
"device_rms": 0.635178488488086
|
||||
},
|
||||
"v_new_17": {
|
||||
"relative_l2": 0.006222788545769237,
|
||||
"max_abs_error": 0.013944387435913086,
|
||||
"source_rms": 0.5384818316759848,
|
||||
"device_rms": 0.5378497636776735
|
||||
},
|
||||
"k_new_18": {
|
||||
"relative_l2": 0.0031373787619186698,
|
||||
"max_abs_error": 0.007158517837524414,
|
||||
"source_rms": 0.7283659936841244,
|
||||
"device_rms": 0.728243335115974
|
||||
},
|
||||
"v_new_18": {
|
||||
"relative_l2": 0.004395458652045999,
|
||||
"max_abs_error": 0.010402202606201172,
|
||||
"source_rms": 0.6382820941879574,
|
||||
"device_rms": 0.6379540669405782
|
||||
},
|
||||
"k_new_19": {
|
||||
"relative_l2": 0.003915531857301537,
|
||||
"max_abs_error": 0.008389949798583984,
|
||||
"source_rms": 0.5790109825658509,
|
||||
"device_rms": 0.5790350688209026
|
||||
},
|
||||
"v_new_19": {
|
||||
"relative_l2": 0.00443498629491144,
|
||||
"max_abs_error": 0.01112210750579834,
|
||||
"source_rms": 0.7220926350353908,
|
||||
"device_rms": 0.721899995335209
|
||||
}
|
||||
},
|
||||
"cache_max_relative_l2": 0.012347926009067344,
|
||||
"limits": [
|
||||
"The historical ONNX export does not attest the exact source checkpoint revision.",
|
||||
"One retained decode fixture cannot establish speech-token or generated-audio quality.",
|
||||
"Actual NPU placement requires a device profile; a request or QNN artifact alone is insufficient.",
|
||||
"Thinker, audio/vision encoders, vocoder, state loop and complete Omni request were not tested."
|
||||
]
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"retrieved_utc": "2026-09-23T23:37:20.477702+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"job_id": "jp2ryv3qg",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jp2ryv3qg/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"source_model_id": "mn4gjjpvn",
|
||||
"target_model_id": "mqp411wgq",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S24",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8gen3",
|
||||
"chipset:sm8650",
|
||||
"hexagon:v75",
|
||||
"soc-model:57",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"options": "--target_runtime qnn_context_binary --quantize_full_type float16",
|
||||
"hub_version": "aihub-2026.08.28.1"
|
||||
}
|
||||
BIN
Binary file not shown.
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:37:27.853385+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "jp0m0646g",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jp0m0646g/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"options": "",
|
||||
"hub_version": "aihub-2026.08.28.1",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S24",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8gen3",
|
||||
"chipset:sm8650",
|
||||
"hexagon:v75",
|
||||
"soc-model:57",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"model_id": "mqp411wgq",
|
||||
"input_dataset_id": "d7zn8rk57",
|
||||
"outputs": "device_output.npz"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:37:21.006320+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "jp2ryv3qg",
|
||||
"target_model_id": "mqp411wgq",
|
||||
"input_dataset_id": "d7zn8rk57",
|
||||
"device": "Samsung Galaxy S24",
|
||||
"options": "",
|
||||
"kind": "inference",
|
||||
"status": "HISTORICAL",
|
||||
"job_id": "jp0m0646g",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jp0m0646g/"
|
||||
}
|
||||
+305
@@ -0,0 +1,305 @@
|
||||
{
|
||||
"scope": "one fixed-cache 20-layer MiniCPM-o speech-head decode step; no complete multimodal stream",
|
||||
"status": "component_inference_numeric_measured",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S25",
|
||||
"os": "15",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8-elite-for-galaxy",
|
||||
"chipset:sm8750-ac",
|
||||
"hexagon:v79",
|
||||
"soc-model:69",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"compile_job_id": "jp4yr9j1p",
|
||||
"inference_job_id": "jpe7l8r75",
|
||||
"source_model_id": "mn4gjjpvn",
|
||||
"target_model_id": "mn7zvvw8m",
|
||||
"input_dataset_id": "d7gwxe3y2",
|
||||
"requested_run_options": "--compute_unit npu",
|
||||
"files": {
|
||||
"source_onnx": {
|
||||
"bytes": 776341465,
|
||||
"sha256": "78cf64804f11ad269ee4180da61588ccc5eefe2942773227345499756e4cc229"
|
||||
},
|
||||
"fixture": {
|
||||
"bytes": 29247541,
|
||||
"sha256": "65919ea1f351feec33c1415a3e3db76d60bc9c2beb3d49a72af75dcb418e83b4"
|
||||
},
|
||||
"cpu_reference": {
|
||||
"bytes": 148471,
|
||||
"sha256": "902adcceadbe761d938ef2ade51cf953c8917ea1abfaee33a10c84e300c1bd96"
|
||||
},
|
||||
"device_output": {
|
||||
"bytes": 105337,
|
||||
"sha256": "633f599b6fb48ba0153a6602024578e9b132f12a7da80f30d6ab45a6b4b5bd9c"
|
||||
}
|
||||
},
|
||||
"tensor_comparison": {
|
||||
"logits": {
|
||||
"relative_l2": 0.003604817571512143,
|
||||
"max_abs_error": 0.01849222183227539,
|
||||
"source_rms": 1.4974441755558239,
|
||||
"device_rms": 1.4975815520684457,
|
||||
"cpu_top1": 1867,
|
||||
"device_top1": 1867,
|
||||
"top1_matches": true
|
||||
},
|
||||
"k_new_0": {
|
||||
"relative_l2": 0.0004768420017180534,
|
||||
"max_abs_error": 0.0030488967895507812,
|
||||
"source_rms": 1.307108108274551,
|
||||
"device_rms": 1.3070583044990185
|
||||
},
|
||||
"v_new_0": {
|
||||
"relative_l2": 0.0003603126630224419,
|
||||
"max_abs_error": 0.0008350610733032227,
|
||||
"source_rms": 0.5083994128636046,
|
||||
"device_rms": 0.5083566801896876
|
||||
},
|
||||
"k_new_1": {
|
||||
"relative_l2": 0.0006006014621578228,
|
||||
"max_abs_error": 0.0020585060119628906,
|
||||
"source_rms": 0.7542700462780494,
|
||||
"device_rms": 0.7542847997905763
|
||||
},
|
||||
"v_new_1": {
|
||||
"relative_l2": 0.0009850478122218705,
|
||||
"max_abs_error": 0.0004919767379760742,
|
||||
"source_rms": 0.0662130257652199,
|
||||
"device_rms": 0.06622593757092551
|
||||
},
|
||||
"k_new_2": {
|
||||
"relative_l2": 0.0013918188255231962,
|
||||
"max_abs_error": 0.005944669246673584,
|
||||
"source_rms": 0.9817805619626084,
|
||||
"device_rms": 0.9819333562222694
|
||||
},
|
||||
"v_new_2": {
|
||||
"relative_l2": 0.0014414899900921244,
|
||||
"max_abs_error": 0.0015971064567565918,
|
||||
"source_rms": 0.24438175712240315,
|
||||
"device_rms": 0.24441703570813905
|
||||
},
|
||||
"k_new_3": {
|
||||
"relative_l2": 0.004245937190563573,
|
||||
"max_abs_error": 0.013946861028671265,
|
||||
"source_rms": 0.9312003114105484,
|
||||
"device_rms": 0.9324225800439694
|
||||
},
|
||||
"v_new_3": {
|
||||
"relative_l2": 0.004898768541682076,
|
||||
"max_abs_error": 0.007790923118591309,
|
||||
"source_rms": 0.27179250329538757,
|
||||
"device_rms": 0.2722731554069219
|
||||
},
|
||||
"k_new_4": {
|
||||
"relative_l2": 0.008357319683860935,
|
||||
"max_abs_error": 0.04427981376647949,
|
||||
"source_rms": 1.2078244698126999,
|
||||
"device_rms": 1.2119446015796154
|
||||
},
|
||||
"v_new_4": {
|
||||
"relative_l2": 0.008863727559428805,
|
||||
"max_abs_error": 0.01918613910675049,
|
||||
"source_rms": 0.4929021962159226,
|
||||
"device_rms": 0.4943822582692528
|
||||
},
|
||||
"k_new_5": {
|
||||
"relative_l2": 0.008330344487322188,
|
||||
"max_abs_error": 0.050071775913238525,
|
||||
"source_rms": 1.0960093751879882,
|
||||
"device_rms": 1.0973710655993856
|
||||
},
|
||||
"v_new_5": {
|
||||
"relative_l2": 0.013109848276876808,
|
||||
"max_abs_error": 0.029880285263061523,
|
||||
"source_rms": 0.49114616760529134,
|
||||
"device_rms": 0.48953354869135374
|
||||
},
|
||||
"k_new_6": {
|
||||
"relative_l2": 0.0066676340170287555,
|
||||
"max_abs_error": 0.037552088499069214,
|
||||
"source_rms": 1.1726536779527166,
|
||||
"device_rms": 1.1736265592360764
|
||||
},
|
||||
"v_new_6": {
|
||||
"relative_l2": 0.010073821087984932,
|
||||
"max_abs_error": 0.017741471529006958,
|
||||
"source_rms": 0.5369044199454948,
|
||||
"device_rms": 0.5363731493205584
|
||||
},
|
||||
"k_new_7": {
|
||||
"relative_l2": 0.004341549883609542,
|
||||
"max_abs_error": 0.024735212326049805,
|
||||
"source_rms": 1.6917150856043028,
|
||||
"device_rms": 1.6947307092332442
|
||||
},
|
||||
"v_new_7": {
|
||||
"relative_l2": 0.007366719463441833,
|
||||
"max_abs_error": 0.02075248956680298,
|
||||
"source_rms": 0.8116460286615261,
|
||||
"device_rms": 0.8109266332812161
|
||||
},
|
||||
"k_new_8": {
|
||||
"relative_l2": 0.005286973065893826,
|
||||
"max_abs_error": 0.0369260311126709,
|
||||
"source_rms": 1.5021910590457033,
|
||||
"device_rms": 1.5038957470210674
|
||||
},
|
||||
"v_new_8": {
|
||||
"relative_l2": 0.007216100089154597,
|
||||
"max_abs_error": 0.030480623245239258,
|
||||
"source_rms": 0.8807572129778053,
|
||||
"device_rms": 0.8793738751401555
|
||||
},
|
||||
"k_new_9": {
|
||||
"relative_l2": 0.005219384806630436,
|
||||
"max_abs_error": 0.019105911254882812,
|
||||
"source_rms": 1.1054146201792987,
|
||||
"device_rms": 1.105981524933566
|
||||
},
|
||||
"v_new_9": {
|
||||
"relative_l2": 0.008339513600691198,
|
||||
"max_abs_error": 0.019251316785812378,
|
||||
"source_rms": 0.6580213796976341,
|
||||
"device_rms": 0.6568745718976601
|
||||
},
|
||||
"k_new_10": {
|
||||
"relative_l2": 0.004872420258791068,
|
||||
"max_abs_error": 0.025604248046875,
|
||||
"source_rms": 1.325937063058518,
|
||||
"device_rms": 1.3271226818646766
|
||||
},
|
||||
"v_new_10": {
|
||||
"relative_l2": 0.007384561351514398,
|
||||
"max_abs_error": 0.022746920585632324,
|
||||
"source_rms": 0.848351970479369,
|
||||
"device_rms": 0.8469938514713699
|
||||
},
|
||||
"k_new_11": {
|
||||
"relative_l2": 0.0036684327450732055,
|
||||
"max_abs_error": 0.007944494485855103,
|
||||
"source_rms": 0.6808336872062942,
|
||||
"device_rms": 0.6808370240599788
|
||||
},
|
||||
"v_new_11": {
|
||||
"relative_l2": 0.006657959894529231,
|
||||
"max_abs_error": 0.00893789529800415,
|
||||
"source_rms": 0.3676426367961426,
|
||||
"device_rms": 0.367326051913896
|
||||
},
|
||||
"k_new_12": {
|
||||
"relative_l2": 0.00437519967357564,
|
||||
"max_abs_error": 0.009633630514144897,
|
||||
"source_rms": 0.6128882051701765,
|
||||
"device_rms": 0.6123538590621596
|
||||
},
|
||||
"v_new_12": {
|
||||
"relative_l2": 0.006266047677935625,
|
||||
"max_abs_error": 0.00861978530883789,
|
||||
"source_rms": 0.34029547623567724,
|
||||
"device_rms": 0.3400227724780066
|
||||
},
|
||||
"k_new_13": {
|
||||
"relative_l2": 0.004275154538917279,
|
||||
"max_abs_error": 0.009413182735443115,
|
||||
"source_rms": 0.613775761616868,
|
||||
"device_rms": 0.6136834702480352
|
||||
},
|
||||
"v_new_13": {
|
||||
"relative_l2": 0.006883031023112881,
|
||||
"max_abs_error": 0.009202957153320312,
|
||||
"source_rms": 0.36907442918730654,
|
||||
"device_rms": 0.3686019889402109
|
||||
},
|
||||
"k_new_14": {
|
||||
"relative_l2": 0.004250477212685874,
|
||||
"max_abs_error": 0.010796785354614258,
|
||||
"source_rms": 0.5980841827735426,
|
||||
"device_rms": 0.5979563556657308
|
||||
},
|
||||
"v_new_14": {
|
||||
"relative_l2": 0.00644672998650109,
|
||||
"max_abs_error": 0.007135152816772461,
|
||||
"source_rms": 0.3506656009170703,
|
||||
"device_rms": 0.35045097633122835
|
||||
},
|
||||
"k_new_15": {
|
||||
"relative_l2": 0.0038119071783685736,
|
||||
"max_abs_error": 0.008935302495956421,
|
||||
"source_rms": 0.622821559168189,
|
||||
"device_rms": 0.6226074020851996
|
||||
},
|
||||
"v_new_15": {
|
||||
"relative_l2": 0.005696997788414253,
|
||||
"max_abs_error": 0.008269354701042175,
|
||||
"source_rms": 0.41823950764909973,
|
||||
"device_rms": 0.4180475728513023
|
||||
},
|
||||
"k_new_16": {
|
||||
"relative_l2": 0.0034009282403524833,
|
||||
"max_abs_error": 0.007880926132202148,
|
||||
"source_rms": 0.6754924715823191,
|
||||
"device_rms": 0.675547820688721
|
||||
},
|
||||
"v_new_16": {
|
||||
"relative_l2": 0.006121492098191588,
|
||||
"max_abs_error": 0.009961724281311035,
|
||||
"source_rms": 0.4659773620123202,
|
||||
"device_rms": 0.465841914286558
|
||||
},
|
||||
"k_new_17": {
|
||||
"relative_l2": 0.0035368926609331127,
|
||||
"max_abs_error": 0.008532732725143433,
|
||||
"source_rms": 0.6354120615871959,
|
||||
"device_rms": 0.6355318620887297
|
||||
},
|
||||
"v_new_17": {
|
||||
"relative_l2": 0.006202687503032289,
|
||||
"max_abs_error": 0.014676839113235474,
|
||||
"source_rms": 0.5384818316759848,
|
||||
"device_rms": 0.5382853413323491
|
||||
},
|
||||
"k_new_18": {
|
||||
"relative_l2": 0.0032849088357320435,
|
||||
"max_abs_error": 0.009111523628234863,
|
||||
"source_rms": 0.7283659936841244,
|
||||
"device_rms": 0.7286277381853394
|
||||
},
|
||||
"v_new_18": {
|
||||
"relative_l2": 0.004489921307521506,
|
||||
"max_abs_error": 0.009913980960845947,
|
||||
"source_rms": 0.6382820941879574,
|
||||
"device_rms": 0.6383502508936021
|
||||
},
|
||||
"k_new_19": {
|
||||
"relative_l2": 0.004013762961457801,
|
||||
"max_abs_error": 0.007941603660583496,
|
||||
"source_rms": 0.5790109825658509,
|
||||
"device_rms": 0.5793150665763361
|
||||
},
|
||||
"v_new_19": {
|
||||
"relative_l2": 0.00450880261583045,
|
||||
"max_abs_error": 0.011058568954467773,
|
||||
"source_rms": 0.7220926350353908,
|
||||
"device_rms": 0.7223734025466527
|
||||
}
|
||||
},
|
||||
"cache_max_relative_l2": 0.013109848276876808,
|
||||
"limits": [
|
||||
"The historical ONNX export does not attest the exact source checkpoint revision.",
|
||||
"One retained decode fixture cannot establish speech-token or generated-audio quality.",
|
||||
"Actual NPU placement requires a device profile; a request or QNN artifact alone is insufficient.",
|
||||
"Thinker, audio/vision encoders, vocoder, state loop and complete Omni request were not tested."
|
||||
]
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"retrieved_utc": "2026-09-23T23:37:17.882748+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"job_id": "jp4yr9j1p",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jp4yr9j1p/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"source_model_id": "mn4gjjpvn",
|
||||
"target_model_id": "mn7zvvw8m",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S25",
|
||||
"os": "15",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8-elite-for-galaxy",
|
||||
"chipset:sm8750-ac",
|
||||
"hexagon:v79",
|
||||
"soc-model:69",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"options": "--target_runtime qnn_context_binary --quantize_full_type float16",
|
||||
"hub_version": "aihub-2026.08.28.1"
|
||||
}
|
||||
BIN
Binary file not shown.
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:37:23.059290+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "jpe7l8r75",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jpe7l8r75/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"options": "--compute_unit npu",
|
||||
"hub_version": "aihub-2026.09.11.0",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S25",
|
||||
"os": "15",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8-elite-for-galaxy",
|
||||
"chipset:sm8750-ac",
|
||||
"hexagon:v79",
|
||||
"soc-model:69",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"model_id": "mn7zvvw8m",
|
||||
"input_dataset_id": "d7gwxe3y2",
|
||||
"outputs": "device_output.npz"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:37:18.432489+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "jp4yr9j1p",
|
||||
"target_model_id": "mn7zvvw8m",
|
||||
"input_dataset_id": "d7gwxe3y2",
|
||||
"device": "Samsung Galaxy S25",
|
||||
"options": "--compute_unit npu",
|
||||
"kind": "inference",
|
||||
"status": "HISTORICAL",
|
||||
"job_id": "jpe7l8r75",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jpe7l8r75/"
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"retrieved_utc": "2026-09-23T23:42:03.580022+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"job_id": "jpe7zlz15",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jpe7zlz15/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"source_model_id": "mn4gjjpvn",
|
||||
"target_model_id": "mn1lvg1zq",
|
||||
"device": {
|
||||
"name": "SA8775P ADP",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-android",
|
||||
"format:auto",
|
||||
"chipset:qualcomm-sa8775p",
|
||||
"hexagon:v73",
|
||||
"soc-model:52",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"options": "--target_runtime qnn_dlc --qnn_options default_graph_htp_precision=FLOAT16",
|
||||
"hub_version": "aihub-2026.09.11.0"
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:38:22.603955+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_model_id": "mn4gjjpvn",
|
||||
"source_sha256": "78cf64804f11ad269ee4180da61588ccc5eefe2942773227345499756e4cc229",
|
||||
"device": "SA8775P ADP",
|
||||
"options": "--target_runtime qnn_dlc --qnn_options default_graph_htp_precision=FLOAT16",
|
||||
"kind": "compile",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jpe7zlz15",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jpe7zlz15/"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:42:04.724186+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "jpe7zlz15",
|
||||
"target_model_id": "mn1lvg1zq",
|
||||
"input_dataset_id": "d7gwxe3y2",
|
||||
"device": "SA8775P ADP",
|
||||
"options": "--compute_unit npu",
|
||||
"kind": "inference",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jp3z9wol5",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jp3z9wol5/"
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"retrieved_utc": "2026-09-23T23:42:05.821498+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"job_id": "jgzlm4mk5",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jgzlm4mk5/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"source_model_id": "mn4gjjpvn",
|
||||
"target_model_id": "mmd04pj3n",
|
||||
"device": {
|
||||
"name": "Snapdragon X Elite CRD",
|
||||
"os": "11",
|
||||
"attributes": [
|
||||
"os:windows",
|
||||
"format:compute",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-windows",
|
||||
"chipset:qualcomm-snapdragon-x-elite",
|
||||
"chipset:sc8380xp",
|
||||
"hexagon:v73",
|
||||
"soc-model:60",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"options": "--target_runtime qnn_dlc --qnn_options default_graph_htp_precision=FLOAT16",
|
||||
"hub_version": "aihub-2026.09.11.0"
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:38:24.348058+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_model_id": "mn4gjjpvn",
|
||||
"source_sha256": "78cf64804f11ad269ee4180da61588ccc5eefe2942773227345499756e4cc229",
|
||||
"device": "Snapdragon X Elite CRD",
|
||||
"options": "--target_runtime qnn_dlc --qnn_options default_graph_htp_precision=FLOAT16",
|
||||
"kind": "compile",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jgzlm4mk5",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jgzlm4mk5/"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:42:06.955300+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "jgzlm4mk5",
|
||||
"target_model_id": "mmd04pj3n",
|
||||
"input_dataset_id": "d7gwxe3y2",
|
||||
"device": "Snapdragon X Elite CRD",
|
||||
"options": "--compute_unit npu",
|
||||
"kind": "inference",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jgolr4dxg",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jgolr4dxg/"
|
||||
}
|
||||
+5
-3
@@ -4,8 +4,10 @@ This is **one fixed-cache-bucket talker decode step**, not complete Qwen3-TTS ge
|
||||
|
||||
The deterministic synthetic input dataset `d7dw6pgy2` has SHA256 `0bf538e1b64dfc9737dc4f925d61c6e4ffa5075a5dcd7afb1e85ac3bd2d9afea`. A local ONNX Runtime 1.29 CPU reference output has SHA256 `ee0ead70e33c02c7d3d6bcb757c5cf0fcbb9eb6fc75c34abfbe8a4479657d5be`. The fixture exercises the stated tensor shapes, but does not represent a real text-to-speech request or establish token/audio quality.
|
||||
|
||||
Historical FP16 QNN context-binary [S24](s24/compile_report.json), [S25](s25/compile_report.json), [SA8775P](sa8775p/compile_report.json), and [Snapdragon X Elite](xelite/compile_report.json) compile jobs succeeded. Their same-fixture NPU inference jobs were [submitted](s24/inference_submission.json), [submitted](s25/inference_submission.json), [submitted](sa8775p/inference_submission.json), and [submitted](xelite/inference_submission.json), respectively; none had a terminal result at this snapshot. The [RB3 compile](rb3/compile_report.json) rejected the floating-point input on this device. These are artifact-specific compile dispositions, not full-model support claims.
|
||||
Historical FP16 QNN context-binary [S24](s24/compile_report.json), [S25](s25/compile_report.json), [SA8775P](sa8775p/compile_report.json), and [Snapdragon X Elite](xelite/compile_report.json) compile jobs succeeded. Same-fixture NPU-requested inference succeeded on all four, but **all changed the CPU reference top-1 logit** on this synthetic step: CPU token 80 versus S24/SA/X Elite 1720 and S25 412. The [S24](s24/audit_report.json), [S25](s25/audit_report.json), [SA8775P](sa8775p/audit_report.json) and [X Elite](xelite/audit_report.json) audits report logits relative L2 **9.33%/8.99%/9.43%/8.57%**, respectively, with maximum K/V relative L2 **11.23%/11.71%/11.54%/11.50%**. These FP16 artifacts are numerically unqualified for this fixture; a successful graph execution is not speech-token parity. The [RB3 compile](rb3/compile_report.json) rejected floating-point input on this device. These are artifact-specific outcomes, not full-model support claims.
|
||||
|
||||
The [collector](../../../../experiments/collect_aihub_component.py) retrieves terminal inference outputs without resubmission. The [tensor auditor](../../../../experiments/audit_qwen_tts_talker_qualcomm.py) enforces the source, fixture, compile target, exact device, output shape and finiteness contracts, and reports logits top-1 plus relative error for all 56 new K/V tensors. It does not convert those metrics to a quality pass without an explicit token/audio tolerance. Its 57-output contract was exercised against the pinned CPU reference as a local self-check; that self-check is not device evidence.
|
||||
Explicit [X Elite ONNX CPU](xelite/cpu_fallback/compile_submission.json) and [SA8775P TFLite CPU](sa8775p/cpu_fallback/compile_submission.json) fallback compiles were submitted from the same source to test separate numerical routes. Their results were pending at this snapshot; neither is an NPU pass or a full TTS deployment.
|
||||
|
||||
Next: retrieve terminal inference tensors, compare logits and every K/V output against the pinned CPU reference with an explicit tolerance, inspect actual placement, then profile a quality-passing decode step. Full device-local talker/predictor/vocoder streaming and power/memory qualification remain separate gates.
|
||||
The [collector](../../../../experiments/collect_aihub_component.py) retrieves terminal inference outputs without resubmission. The [tensor auditor](../../../../experiments/audit_qwen_tts_talker_qualcomm.py) enforces the source, fixture, compile target, exact device, output shape and finiteness contracts, and reports logits top-1 plus relative error for all 56 new K/V tensors. It does not convert those metrics to a quality pass without an explicit token/audio tolerance. Its 57-output contract was exercised against the pinned CPU reference as a local self-check; that self-check is not device evidence. Requested NPU placement still needs a profile.
|
||||
|
||||
Next: investigate FP16 numerical drift using realistic generated-cache fixtures or a higher-precision/mixed-precision artifact, and inspect placement only for a quality-passing decode step. Full device-local talker/predictor/vocoder streaming and power/memory qualification remain separate gates.
|
||||
+401
@@ -0,0 +1,401 @@
|
||||
{
|
||||
"scope": "one synthetic fixed-cache 28-layer Qwen3-TTS talker decode step; no complete TTS stream",
|
||||
"status": "component_executed_numeric_measured",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S24",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8gen3",
|
||||
"chipset:sm8650",
|
||||
"hexagon:v75",
|
||||
"soc-model:57",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"compile_job_id": "jpvdx28zp",
|
||||
"inference_job_id": "j5687dyng",
|
||||
"source_model_id": "mq3xre4rq",
|
||||
"target_model_id": "mnowr3epq",
|
||||
"input_dataset_id": "d7dw6pgy2",
|
||||
"requested_run_options": "--compute_unit npu",
|
||||
"files": {
|
||||
"source_onnx": {
|
||||
"bytes": 1776377476,
|
||||
"sha256": "41feb08517678e94d9301e992d0df610fc60e7c9faba5600703298c86554fd22"
|
||||
},
|
||||
"fixture": {
|
||||
"bytes": 54591500,
|
||||
"sha256": "0bf538e1b64dfc9737dc4f925d61c6e4ffa5075a5dcd7afb1e85ac3bd2d9afea"
|
||||
},
|
||||
"cpu_reference": {
|
||||
"bytes": 237615,
|
||||
"sha256": "ee0ead70e33c02c7d3d6bcb757c5cf0fcbb9eb6fc75c34abfbe8a4479657d5be"
|
||||
},
|
||||
"device_output": {
|
||||
"bytes": 168735,
|
||||
"sha256": "382dcfab474bd9900e52217f1797cff570a160b8f3bfcb07d73f4f83cfdf7769"
|
||||
}
|
||||
},
|
||||
"tensor_comparison": {
|
||||
"logits": {
|
||||
"relative_l2": 0.09326798445788166,
|
||||
"max_abs_error": 1.482887178659439,
|
||||
"reference_rms": 3.7322664515389006,
|
||||
"output_rms": 3.7421461231693893,
|
||||
"cpu_top1": 80,
|
||||
"device_top1": 1720,
|
||||
"top1_matches": false
|
||||
},
|
||||
"k_new_0": {
|
||||
"relative_l2": 0.0005229822431162853,
|
||||
"max_abs_error": 0.003747701644897461,
|
||||
"reference_rms": 1.1349352034374294,
|
||||
"output_rms": 1.1348869276631997
|
||||
},
|
||||
"v_new_0": {
|
||||
"relative_l2": 0.0003622125877911652,
|
||||
"max_abs_error": 0.00020897388458251953,
|
||||
"reference_rms": 0.1383137764927112,
|
||||
"output_rms": 0.1383144487129386
|
||||
},
|
||||
"k_new_1": {
|
||||
"relative_l2": 0.0015685664713715498,
|
||||
"max_abs_error": 0.008643865585327148,
|
||||
"reference_rms": 1.4773734279001252,
|
||||
"output_rms": 1.477313569142581
|
||||
},
|
||||
"v_new_1": {
|
||||
"relative_l2": 0.0015784708023866055,
|
||||
"max_abs_error": 0.0008470192551612854,
|
||||
"reference_rms": 0.162893856079411,
|
||||
"output_rms": 0.16291373133977713
|
||||
},
|
||||
"k_new_2": {
|
||||
"relative_l2": 0.002714729423457892,
|
||||
"max_abs_error": 0.016630858182907104,
|
||||
"reference_rms": 1.4749251890370425,
|
||||
"output_rms": 1.4749108022195403
|
||||
},
|
||||
"v_new_2": {
|
||||
"relative_l2": 0.003065088283861766,
|
||||
"max_abs_error": 0.002920284867286682,
|
||||
"reference_rms": 0.2996575863196337,
|
||||
"output_rms": 0.29956709759675754
|
||||
},
|
||||
"k_new_3": {
|
||||
"relative_l2": 0.0046630057556674865,
|
||||
"max_abs_error": 0.03286319971084595,
|
||||
"reference_rms": 1.4529848472712377,
|
||||
"output_rms": 1.4528223150721002
|
||||
},
|
||||
"v_new_3": {
|
||||
"relative_l2": 0.0046316789003247785,
|
||||
"max_abs_error": 0.005253523588180542,
|
||||
"reference_rms": 0.2917197159228712,
|
||||
"output_rms": 0.29150557288580126
|
||||
},
|
||||
"k_new_4": {
|
||||
"relative_l2": 0.0053651537457211455,
|
||||
"max_abs_error": 0.043274760246276855,
|
||||
"reference_rms": 1.5413980317857385,
|
||||
"output_rms": 1.5416385591539468
|
||||
},
|
||||
"v_new_4": {
|
||||
"relative_l2": 0.007284049635712301,
|
||||
"max_abs_error": 0.013043045997619629,
|
||||
"reference_rms": 0.41222657123793377,
|
||||
"output_rms": 0.4112516421200526
|
||||
},
|
||||
"k_new_5": {
|
||||
"relative_l2": 0.0069512706655330235,
|
||||
"max_abs_error": 0.05223953723907471,
|
||||
"reference_rms": 1.6285454176549035,
|
||||
"output_rms": 1.628551180269626
|
||||
},
|
||||
"v_new_5": {
|
||||
"relative_l2": 0.00855076287121217,
|
||||
"max_abs_error": 0.010519325733184814,
|
||||
"reference_rms": 0.32638470026742417,
|
||||
"output_rms": 0.32593994876819316
|
||||
},
|
||||
"k_new_6": {
|
||||
"relative_l2": 0.009509550016354701,
|
||||
"max_abs_error": 0.09950661659240723,
|
||||
"reference_rms": 1.8000231156167423,
|
||||
"output_rms": 1.8004271490288184
|
||||
},
|
||||
"v_new_6": {
|
||||
"relative_l2": 0.0116848409751892,
|
||||
"max_abs_error": 0.013239085674285889,
|
||||
"reference_rms": 0.2629708328253682,
|
||||
"output_rms": 0.26270997499973897
|
||||
},
|
||||
"k_new_7": {
|
||||
"relative_l2": 0.011177581166086912,
|
||||
"max_abs_error": 0.1820392608642578,
|
||||
"reference_rms": 1.6255722851184644,
|
||||
"output_rms": 1.6256969797890533
|
||||
},
|
||||
"v_new_7": {
|
||||
"relative_l2": 0.012057046774025372,
|
||||
"max_abs_error": 0.016807354986667633,
|
||||
"reference_rms": 0.348368074126628,
|
||||
"output_rms": 0.3481756067543749
|
||||
},
|
||||
"k_new_8": {
|
||||
"relative_l2": 0.01313724737439118,
|
||||
"max_abs_error": 0.08209741115570068,
|
||||
"reference_rms": 1.516512353065435,
|
||||
"output_rms": 1.5162749009216188
|
||||
},
|
||||
"v_new_8": {
|
||||
"relative_l2": 0.013758015043589646,
|
||||
"max_abs_error": 0.014397352933883667,
|
||||
"reference_rms": 0.28027916002007797,
|
||||
"output_rms": 0.2804472337112763
|
||||
},
|
||||
"k_new_9": {
|
||||
"relative_l2": 0.015591594158960529,
|
||||
"max_abs_error": 0.10860276222229004,
|
||||
"reference_rms": 1.5960043895576705,
|
||||
"output_rms": 1.5968098629532432
|
||||
},
|
||||
"v_new_9": {
|
||||
"relative_l2": 0.019815061331476244,
|
||||
"max_abs_error": 0.02331852912902832,
|
||||
"reference_rms": 0.37541008682864946,
|
||||
"output_rms": 0.37496705260272706
|
||||
},
|
||||
"k_new_10": {
|
||||
"relative_l2": 0.020425541036890676,
|
||||
"max_abs_error": 0.15811896324157715,
|
||||
"reference_rms": 1.6561161468173449,
|
||||
"output_rms": 1.6555274020140316
|
||||
},
|
||||
"v_new_10": {
|
||||
"relative_l2": 0.027826594697380313,
|
||||
"max_abs_error": 0.030216187238693237,
|
||||
"reference_rms": 0.26429778504066537,
|
||||
"output_rms": 0.2647061980579945
|
||||
},
|
||||
"k_new_11": {
|
||||
"relative_l2": 0.024578001055724375,
|
||||
"max_abs_error": 0.14571982622146606,
|
||||
"reference_rms": 1.5170462775366995,
|
||||
"output_rms": 1.5163429308945129
|
||||
},
|
||||
"v_new_11": {
|
||||
"relative_l2": 0.033142200134693,
|
||||
"max_abs_error": 0.028120774717535824,
|
||||
"reference_rms": 0.22548908846990579,
|
||||
"output_rms": 0.22595584013339062
|
||||
},
|
||||
"k_new_12": {
|
||||
"relative_l2": 0.0243001988598747,
|
||||
"max_abs_error": 0.13294774293899536,
|
||||
"reference_rms": 1.5731131793440647,
|
||||
"output_rms": 1.5728717894778108
|
||||
},
|
||||
"v_new_12": {
|
||||
"relative_l2": 0.03207291372477802,
|
||||
"max_abs_error": 0.023222461342811584,
|
||||
"reference_rms": 0.20839677682266325,
|
||||
"output_rms": 0.2087994717849238
|
||||
},
|
||||
"k_new_13": {
|
||||
"relative_l2": 0.026335992425878422,
|
||||
"max_abs_error": 0.15324020385742188,
|
||||
"reference_rms": 1.4939452330557514,
|
||||
"output_rms": 1.493167185846336
|
||||
},
|
||||
"v_new_13": {
|
||||
"relative_l2": 0.035032893433589125,
|
||||
"max_abs_error": 0.029594309628009796,
|
||||
"reference_rms": 0.24929462665621008,
|
||||
"output_rms": 0.2499611556480011
|
||||
},
|
||||
"k_new_14": {
|
||||
"relative_l2": 0.0273760395469954,
|
||||
"max_abs_error": 0.15425953269004822,
|
||||
"reference_rms": 1.4942429579997731,
|
||||
"output_rms": 1.4938717305991878
|
||||
},
|
||||
"v_new_14": {
|
||||
"relative_l2": 0.041116967961074544,
|
||||
"max_abs_error": 0.04979333281517029,
|
||||
"reference_rms": 0.32824019866053383,
|
||||
"output_rms": 0.3296733925864719
|
||||
},
|
||||
"k_new_15": {
|
||||
"relative_l2": 0.03268151193421523,
|
||||
"max_abs_error": 0.18472003936767578,
|
||||
"reference_rms": 1.6561072116211841,
|
||||
"output_rms": 1.6549968573829799
|
||||
},
|
||||
"v_new_15": {
|
||||
"relative_l2": 0.04721001246535048,
|
||||
"max_abs_error": 0.04259920120239258,
|
||||
"reference_rms": 0.24976283574047556,
|
||||
"output_rms": 0.25271063257497606
|
||||
},
|
||||
"k_new_16": {
|
||||
"relative_l2": 0.043354789106215055,
|
||||
"max_abs_error": 0.29831528663635254,
|
||||
"reference_rms": 1.617983543941328,
|
||||
"output_rms": 1.6165859133343514
|
||||
},
|
||||
"v_new_16": {
|
||||
"relative_l2": 0.05715941094723187,
|
||||
"max_abs_error": 0.08700801432132721,
|
||||
"reference_rms": 0.3826095020438836,
|
||||
"output_rms": 0.38698479796797636
|
||||
},
|
||||
"k_new_17": {
|
||||
"relative_l2": 0.05811792920193554,
|
||||
"max_abs_error": 0.4855295866727829,
|
||||
"reference_rms": 1.6828817809060599,
|
||||
"output_rms": 1.68424535429126
|
||||
},
|
||||
"v_new_17": {
|
||||
"relative_l2": 0.06225801345078975,
|
||||
"max_abs_error": 0.10319682955741882,
|
||||
"reference_rms": 0.4656876133547257,
|
||||
"output_rms": 0.46787685589723477
|
||||
},
|
||||
"k_new_18": {
|
||||
"relative_l2": 0.06076183009186204,
|
||||
"max_abs_error": 0.4048278331756592,
|
||||
"reference_rms": 1.5546463592216835,
|
||||
"output_rms": 1.5533008727359843
|
||||
},
|
||||
"v_new_18": {
|
||||
"relative_l2": 0.07211321265431962,
|
||||
"max_abs_error": 0.11240960657596588,
|
||||
"reference_rms": 0.5200020604619255,
|
||||
"output_rms": 0.5217945561230504
|
||||
},
|
||||
"k_new_19": {
|
||||
"relative_l2": 0.06518122953316716,
|
||||
"max_abs_error": 0.36439695954322815,
|
||||
"reference_rms": 1.5217070844504548,
|
||||
"output_rms": 1.5203145871311905
|
||||
},
|
||||
"v_new_19": {
|
||||
"relative_l2": 0.07880131876348201,
|
||||
"max_abs_error": 0.1700134538114071,
|
||||
"reference_rms": 0.5402920798790236,
|
||||
"output_rms": 0.543892535704153
|
||||
},
|
||||
"k_new_20": {
|
||||
"relative_l2": 0.0803242905205174,
|
||||
"max_abs_error": 0.678443431854248,
|
||||
"reference_rms": 1.6813854859968058,
|
||||
"output_rms": 1.680891611252685
|
||||
},
|
||||
"v_new_20": {
|
||||
"relative_l2": 0.09042271211357834,
|
||||
"max_abs_error": 0.1755550503730774,
|
||||
"reference_rms": 0.6088730895277438,
|
||||
"output_rms": 0.609399398529315
|
||||
},
|
||||
"k_new_21": {
|
||||
"relative_l2": 0.07493021617103167,
|
||||
"max_abs_error": 0.7069820761680603,
|
||||
"reference_rms": 1.7370493419005983,
|
||||
"output_rms": 1.7331228408610584
|
||||
},
|
||||
"v_new_21": {
|
||||
"relative_l2": 0.09713109565480485,
|
||||
"max_abs_error": 0.2576836347579956,
|
||||
"reference_rms": 0.6072011016139925,
|
||||
"output_rms": 0.6069517180097386
|
||||
},
|
||||
"k_new_22": {
|
||||
"relative_l2": 0.09232573383429477,
|
||||
"max_abs_error": 0.9603571891784668,
|
||||
"reference_rms": 1.9661056736696203,
|
||||
"output_rms": 1.9643726019159282
|
||||
},
|
||||
"v_new_22": {
|
||||
"relative_l2": 0.10308435392479011,
|
||||
"max_abs_error": 0.2859683260321617,
|
||||
"reference_rms": 0.7362191931025958,
|
||||
"output_rms": 0.738890591131214
|
||||
},
|
||||
"k_new_23": {
|
||||
"relative_l2": 0.09072314938741241,
|
||||
"max_abs_error": 0.5558215975761414,
|
||||
"reference_rms": 1.6005997999291703,
|
||||
"output_rms": 1.5963890082747223
|
||||
},
|
||||
"v_new_23": {
|
||||
"relative_l2": 0.11225282955267173,
|
||||
"max_abs_error": 0.2807217836380005,
|
||||
"reference_rms": 0.6675198669572269,
|
||||
"output_rms": 0.6582956851999205
|
||||
},
|
||||
"k_new_24": {
|
||||
"relative_l2": 0.08933652886984028,
|
||||
"max_abs_error": 0.6308331489562988,
|
||||
"reference_rms": 1.6337684549178757,
|
||||
"output_rms": 1.640645072533566
|
||||
},
|
||||
"v_new_24": {
|
||||
"relative_l2": 0.11098970336690335,
|
||||
"max_abs_error": 0.22495862655341625,
|
||||
"reference_rms": 0.6240472225015637,
|
||||
"output_rms": 0.6259650287171282
|
||||
},
|
||||
"k_new_25": {
|
||||
"relative_l2": 0.07807886029557917,
|
||||
"max_abs_error": 0.4474449157714844,
|
||||
"reference_rms": 1.527601662385539,
|
||||
"output_rms": 1.5310503434925258
|
||||
},
|
||||
"v_new_25": {
|
||||
"relative_l2": 0.09632050804361386,
|
||||
"max_abs_error": 0.3357442319393158,
|
||||
"reference_rms": 0.9133071626950193,
|
||||
"output_rms": 0.9118248050125254
|
||||
},
|
||||
"k_new_26": {
|
||||
"relative_l2": 0.07427047461000932,
|
||||
"max_abs_error": 0.436142660677433,
|
||||
"reference_rms": 1.4905787414732559,
|
||||
"output_rms": 1.4937458487793318
|
||||
},
|
||||
"v_new_26": {
|
||||
"relative_l2": 0.1116172718037885,
|
||||
"max_abs_error": 0.27048417925834656,
|
||||
"reference_rms": 0.7009595279765859,
|
||||
"output_rms": 0.7048206665130158
|
||||
},
|
||||
"k_new_27": {
|
||||
"relative_l2": 0.09061553874430112,
|
||||
"max_abs_error": 0.5266905277967453,
|
||||
"reference_rms": 1.5453936433107067,
|
||||
"output_rms": 1.5468927295472754
|
||||
},
|
||||
"v_new_27": {
|
||||
"relative_l2": 0.09656697969622459,
|
||||
"max_abs_error": 0.20447054505348206,
|
||||
"reference_rms": 0.6731545154533489,
|
||||
"output_rms": 0.6811216149235594
|
||||
}
|
||||
},
|
||||
"cache_max_relative_l2": 0.11225282955267173,
|
||||
"limits": [
|
||||
"The historical source export does not attest an exact checkpoint revision.",
|
||||
"The fixture is synthetic and has no token-to-audio quality tolerance.",
|
||||
"Requested NPU is not actual placement without a device profile.",
|
||||
"No predictor, vocoder, persistent cache loop or complete TTS stream was tested."
|
||||
]
|
||||
}
|
||||
BIN
Binary file not shown.
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:44:40.837683+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "j5687dyng",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/j5687dyng/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"options": "--compute_unit npu",
|
||||
"hub_version": "aihub-2026.09.11.0",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S24",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8gen3",
|
||||
"chipset:sm8650",
|
||||
"hexagon:v75",
|
||||
"soc-model:57",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"model_id": "mnowr3epq",
|
||||
"input_dataset_id": "d7dw6pgy2",
|
||||
"outputs": "device_output.npz"
|
||||
}
|
||||
+401
@@ -0,0 +1,401 @@
|
||||
{
|
||||
"scope": "one synthetic fixed-cache 28-layer Qwen3-TTS talker decode step; no complete TTS stream",
|
||||
"status": "component_executed_numeric_measured",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S25",
|
||||
"os": "15",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8-elite-for-galaxy",
|
||||
"chipset:sm8750-ac",
|
||||
"hexagon:v79",
|
||||
"soc-model:69",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"compile_job_id": "jgj74391g",
|
||||
"inference_job_id": "jglyx1v25",
|
||||
"source_model_id": "mq3xre4rq",
|
||||
"target_model_id": "mqv3y4xlm",
|
||||
"input_dataset_id": "d7dw6pgy2",
|
||||
"requested_run_options": "--compute_unit npu",
|
||||
"files": {
|
||||
"source_onnx": {
|
||||
"bytes": 1776377476,
|
||||
"sha256": "41feb08517678e94d9301e992d0df610fc60e7c9faba5600703298c86554fd22"
|
||||
},
|
||||
"fixture": {
|
||||
"bytes": 54591500,
|
||||
"sha256": "0bf538e1b64dfc9737dc4f925d61c6e4ffa5075a5dcd7afb1e85ac3bd2d9afea"
|
||||
},
|
||||
"cpu_reference": {
|
||||
"bytes": 237615,
|
||||
"sha256": "ee0ead70e33c02c7d3d6bcb757c5cf0fcbb9eb6fc75c34abfbe8a4479657d5be"
|
||||
},
|
||||
"device_output": {
|
||||
"bytes": 167992,
|
||||
"sha256": "c53d142fac597ef84659f0c1fc2f3116a4043a30f25ace720dd770568156e52e"
|
||||
}
|
||||
},
|
||||
"tensor_comparison": {
|
||||
"logits": {
|
||||
"relative_l2": 0.08987602741927851,
|
||||
"max_abs_error": 1.3871840238571167,
|
||||
"reference_rms": 3.7322664515389006,
|
||||
"output_rms": 3.744265940933905,
|
||||
"cpu_top1": 80,
|
||||
"device_top1": 412,
|
||||
"top1_matches": false
|
||||
},
|
||||
"k_new_0": {
|
||||
"relative_l2": 0.0005230110003561782,
|
||||
"max_abs_error": 0.0037479400634765625,
|
||||
"reference_rms": 1.1349352034374294,
|
||||
"output_rms": 1.1348859234516222
|
||||
},
|
||||
"v_new_0": {
|
||||
"relative_l2": 0.000362212102381853,
|
||||
"max_abs_error": 0.00020900368690490723,
|
||||
"reference_rms": 0.1383137764927112,
|
||||
"output_rms": 0.1383144368591086
|
||||
},
|
||||
"k_new_1": {
|
||||
"relative_l2": 0.001540598700847705,
|
||||
"max_abs_error": 0.0086439847946167,
|
||||
"reference_rms": 1.4773734279001252,
|
||||
"output_rms": 1.4772890926361812
|
||||
},
|
||||
"v_new_1": {
|
||||
"relative_l2": 0.0015613755499334814,
|
||||
"max_abs_error": 0.0009385757148265839,
|
||||
"reference_rms": 0.162893856079411,
|
||||
"output_rms": 0.16291330617946456
|
||||
},
|
||||
"k_new_2": {
|
||||
"relative_l2": 0.002761054571015776,
|
||||
"max_abs_error": 0.020048856735229492,
|
||||
"reference_rms": 1.4749251890370425,
|
||||
"output_rms": 1.4749432597408292
|
||||
},
|
||||
"v_new_2": {
|
||||
"relative_l2": 0.003124315994655873,
|
||||
"max_abs_error": 0.003032371401786804,
|
||||
"reference_rms": 0.2996575863196337,
|
||||
"output_rms": 0.29957990625185127
|
||||
},
|
||||
"k_new_3": {
|
||||
"relative_l2": 0.004821709696188454,
|
||||
"max_abs_error": 0.03383982181549072,
|
||||
"reference_rms": 1.4529848472712377,
|
||||
"output_rms": 1.4528154889915204
|
||||
},
|
||||
"v_new_3": {
|
||||
"relative_l2": 0.004771549667840103,
|
||||
"max_abs_error": 0.00476527214050293,
|
||||
"reference_rms": 0.2917197159228712,
|
||||
"output_rms": 0.2914876080010164
|
||||
},
|
||||
"k_new_4": {
|
||||
"relative_l2": 0.0054442634395963415,
|
||||
"max_abs_error": 0.03994256258010864,
|
||||
"reference_rms": 1.5413980317857385,
|
||||
"output_rms": 1.5416720658043106
|
||||
},
|
||||
"v_new_4": {
|
||||
"relative_l2": 0.007738122847452517,
|
||||
"max_abs_error": 0.015972793102264404,
|
||||
"reference_rms": 0.41222657123793377,
|
||||
"output_rms": 0.4112461648522345
|
||||
},
|
||||
"k_new_5": {
|
||||
"relative_l2": 0.007012562679528545,
|
||||
"max_abs_error": 0.052239418029785156,
|
||||
"reference_rms": 1.6285454176549035,
|
||||
"output_rms": 1.628484434913322
|
||||
},
|
||||
"v_new_5": {
|
||||
"relative_l2": 0.008904990613392486,
|
||||
"max_abs_error": 0.010504394769668579,
|
||||
"reference_rms": 0.32638470026742417,
|
||||
"output_rms": 0.3260051823719569
|
||||
},
|
||||
"k_new_6": {
|
||||
"relative_l2": 0.009850213673870498,
|
||||
"max_abs_error": 0.10601842403411865,
|
||||
"reference_rms": 1.8000231156167423,
|
||||
"output_rms": 1.8004311732968745
|
||||
},
|
||||
"v_new_6": {
|
||||
"relative_l2": 0.011808014241788779,
|
||||
"max_abs_error": 0.012933917343616486,
|
||||
"reference_rms": 0.2629708328253682,
|
||||
"output_rms": 0.26273516655300605
|
||||
},
|
||||
"k_new_7": {
|
||||
"relative_l2": 0.011606622560418728,
|
||||
"max_abs_error": 0.18203949928283691,
|
||||
"reference_rms": 1.6255722851184644,
|
||||
"output_rms": 1.6258290143511915
|
||||
},
|
||||
"v_new_7": {
|
||||
"relative_l2": 0.012365408379984082,
|
||||
"max_abs_error": 0.018272191286087036,
|
||||
"reference_rms": 0.348368074126628,
|
||||
"output_rms": 0.3483496231074009
|
||||
},
|
||||
"k_new_8": {
|
||||
"relative_l2": 0.013251174257627183,
|
||||
"max_abs_error": 0.08795690536499023,
|
||||
"reference_rms": 1.516512353065435,
|
||||
"output_rms": 1.5161379941046276
|
||||
},
|
||||
"v_new_8": {
|
||||
"relative_l2": 0.01388509680167999,
|
||||
"max_abs_error": 0.01561802625656128,
|
||||
"reference_rms": 0.28027916002007797,
|
||||
"output_rms": 0.2806319128941407
|
||||
},
|
||||
"k_new_9": {
|
||||
"relative_l2": 0.015305669641515982,
|
||||
"max_abs_error": 0.10664939880371094,
|
||||
"reference_rms": 1.5960043895576705,
|
||||
"output_rms": 1.5967604839081795
|
||||
},
|
||||
"v_new_9": {
|
||||
"relative_l2": 0.019710199157081793,
|
||||
"max_abs_error": 0.025908350944519043,
|
||||
"reference_rms": 0.37541008682864946,
|
||||
"output_rms": 0.37524481110291996
|
||||
},
|
||||
"k_new_10": {
|
||||
"relative_l2": 0.02098040703507274,
|
||||
"max_abs_error": 0.15421295166015625,
|
||||
"reference_rms": 1.6561161468173449,
|
||||
"output_rms": 1.6556291953279028
|
||||
},
|
||||
"v_new_10": {
|
||||
"relative_l2": 0.02900535491158207,
|
||||
"max_abs_error": 0.034610748291015625,
|
||||
"reference_rms": 0.26429778504066537,
|
||||
"output_rms": 0.26486285964325695
|
||||
},
|
||||
"k_new_11": {
|
||||
"relative_l2": 0.02458741345280285,
|
||||
"max_abs_error": 0.14026427268981934,
|
||||
"reference_rms": 1.5170462775366995,
|
||||
"output_rms": 1.5164885400864097
|
||||
},
|
||||
"v_new_11": {
|
||||
"relative_l2": 0.033385509536275164,
|
||||
"max_abs_error": 0.027840405702590942,
|
||||
"reference_rms": 0.22548908846990579,
|
||||
"output_rms": 0.22611854484395508
|
||||
},
|
||||
"k_new_12": {
|
||||
"relative_l2": 0.024703908253396033,
|
||||
"max_abs_error": 0.14281916618347168,
|
||||
"reference_rms": 1.5731131793440647,
|
||||
"output_rms": 1.5727749020897415
|
||||
},
|
||||
"v_new_12": {
|
||||
"relative_l2": 0.032027777528514816,
|
||||
"max_abs_error": 0.02357383258640766,
|
||||
"reference_rms": 0.20839677682266325,
|
||||
"output_rms": 0.20897931565876085
|
||||
},
|
||||
"k_new_13": {
|
||||
"relative_l2": 0.02727958033858871,
|
||||
"max_abs_error": 0.18449044227600098,
|
||||
"reference_rms": 1.4939452330557514,
|
||||
"output_rms": 1.492896003482396
|
||||
},
|
||||
"v_new_13": {
|
||||
"relative_l2": 0.03536399799198983,
|
||||
"max_abs_error": 0.029655352234840393,
|
||||
"reference_rms": 0.24929462665621008,
|
||||
"output_rms": 0.24965703372611148
|
||||
},
|
||||
"k_new_14": {
|
||||
"relative_l2": 0.028529373509095894,
|
||||
"max_abs_error": 0.16158369183540344,
|
||||
"reference_rms": 1.4942429579997731,
|
||||
"output_rms": 1.4941069885427924
|
||||
},
|
||||
"v_new_14": {
|
||||
"relative_l2": 0.042318536709079396,
|
||||
"max_abs_error": 0.05322727560997009,
|
||||
"reference_rms": 0.32824019866053383,
|
||||
"output_rms": 0.3293045937625869
|
||||
},
|
||||
"k_new_15": {
|
||||
"relative_l2": 0.03318615370726284,
|
||||
"max_abs_error": 0.20718109607696533,
|
||||
"reference_rms": 1.6561072116211841,
|
||||
"output_rms": 1.6554001336128799
|
||||
},
|
||||
"v_new_15": {
|
||||
"relative_l2": 0.04788461259739498,
|
||||
"max_abs_error": 0.04036116600036621,
|
||||
"reference_rms": 0.24976283574047556,
|
||||
"output_rms": 0.2524891712410087
|
||||
},
|
||||
"k_new_16": {
|
||||
"relative_l2": 0.042748005881473046,
|
||||
"max_abs_error": 0.2903108596801758,
|
||||
"reference_rms": 1.617983543941328,
|
||||
"output_rms": 1.6162731649705546
|
||||
},
|
||||
"v_new_16": {
|
||||
"relative_l2": 0.05602842193381644,
|
||||
"max_abs_error": 0.0824914276599884,
|
||||
"reference_rms": 0.3826095020438836,
|
||||
"output_rms": 0.38606969975980593
|
||||
},
|
||||
"k_new_17": {
|
||||
"relative_l2": 0.0573413163063548,
|
||||
"max_abs_error": 0.4741595983505249,
|
||||
"reference_rms": 1.6828817809060599,
|
||||
"output_rms": 1.6837097484064685
|
||||
},
|
||||
"v_new_17": {
|
||||
"relative_l2": 0.06145866718948853,
|
||||
"max_abs_error": 0.10026717185974121,
|
||||
"reference_rms": 0.4656876133547257,
|
||||
"output_rms": 0.46747467593403624
|
||||
},
|
||||
"k_new_18": {
|
||||
"relative_l2": 0.06153641493618332,
|
||||
"max_abs_error": 0.4087338447570801,
|
||||
"reference_rms": 1.5546463592216835,
|
||||
"output_rms": 1.553254647654723
|
||||
},
|
||||
"v_new_18": {
|
||||
"relative_l2": 0.07062542578480387,
|
||||
"max_abs_error": 0.11570566892623901,
|
||||
"reference_rms": 0.5200020604619255,
|
||||
"output_rms": 0.5221170894640569
|
||||
},
|
||||
"k_new_19": {
|
||||
"relative_l2": 0.06594656362523482,
|
||||
"max_abs_error": 0.35194581747055054,
|
||||
"reference_rms": 1.5217070844504548,
|
||||
"output_rms": 1.5198815066442908
|
||||
},
|
||||
"v_new_19": {
|
||||
"relative_l2": 0.07907775854385347,
|
||||
"max_abs_error": 0.1748962700366974,
|
||||
"reference_rms": 0.5402920798790236,
|
||||
"output_rms": 0.5444699991813782
|
||||
},
|
||||
"k_new_20": {
|
||||
"relative_l2": 0.08084185434879565,
|
||||
"max_abs_error": 0.7370367050170898,
|
||||
"reference_rms": 1.6813854859968058,
|
||||
"output_rms": 1.6814891759755295
|
||||
},
|
||||
"v_new_20": {
|
||||
"relative_l2": 0.09227356347540493,
|
||||
"max_abs_error": 0.17555499076843262,
|
||||
"reference_rms": 0.6088730895277438,
|
||||
"output_rms": 0.6084923327462645
|
||||
},
|
||||
"k_new_21": {
|
||||
"relative_l2": 0.07709836006531275,
|
||||
"max_abs_error": 0.6928219795227051,
|
||||
"reference_rms": 1.7370493419005983,
|
||||
"output_rms": 1.7330298811755278
|
||||
},
|
||||
"v_new_21": {
|
||||
"relative_l2": 0.09726844988798349,
|
||||
"max_abs_error": 0.2340153455734253,
|
||||
"reference_rms": 0.6072011016139925,
|
||||
"output_rms": 0.6068723668025157
|
||||
},
|
||||
"k_new_22": {
|
||||
"relative_l2": 0.0904278127368492,
|
||||
"max_abs_error": 0.9759817123413086,
|
||||
"reference_rms": 1.9661056736696203,
|
||||
"output_rms": 1.9641139705255086
|
||||
},
|
||||
"v_new_22": {
|
||||
"relative_l2": 0.10108286614731754,
|
||||
"max_abs_error": 0.2545352280139923,
|
||||
"reference_rms": 0.7362191931025958,
|
||||
"output_rms": 0.7373746230019541
|
||||
},
|
||||
"k_new_23": {
|
||||
"relative_l2": 0.0936142531275267,
|
||||
"max_abs_error": 0.6393176317214966,
|
||||
"reference_rms": 1.6005997999291703,
|
||||
"output_rms": 1.5963565808784597
|
||||
},
|
||||
"v_new_23": {
|
||||
"relative_l2": 0.11386042803999914,
|
||||
"max_abs_error": 0.3141513913869858,
|
||||
"reference_rms": 0.6675198669572269,
|
||||
"output_rms": 0.657863760711424
|
||||
},
|
||||
"k_new_24": {
|
||||
"relative_l2": 0.093063232099094,
|
||||
"max_abs_error": 0.6855204105377197,
|
||||
"reference_rms": 1.6337684549178757,
|
||||
"output_rms": 1.64070012900969
|
||||
},
|
||||
"v_new_24": {
|
||||
"relative_l2": 0.11669454225130346,
|
||||
"max_abs_error": 0.25028255581855774,
|
||||
"reference_rms": 0.6240472225015637,
|
||||
"output_rms": 0.6255317007252221
|
||||
},
|
||||
"k_new_25": {
|
||||
"relative_l2": 0.0820184774861779,
|
||||
"max_abs_error": 0.44124460220336914,
|
||||
"reference_rms": 1.527601662385539,
|
||||
"output_rms": 1.532089031858724
|
||||
},
|
||||
"v_new_25": {
|
||||
"relative_l2": 0.10476823325221819,
|
||||
"max_abs_error": 0.3923848867416382,
|
||||
"reference_rms": 0.9133071626950193,
|
||||
"output_rms": 0.908104215343327
|
||||
},
|
||||
"k_new_26": {
|
||||
"relative_l2": 0.07596530574012564,
|
||||
"max_abs_error": 0.4392259120941162,
|
||||
"reference_rms": 1.4905787414732559,
|
||||
"output_rms": 1.4932524693543205
|
||||
},
|
||||
"v_new_26": {
|
||||
"relative_l2": 0.11714982511905245,
|
||||
"max_abs_error": 0.28964920341968536,
|
||||
"reference_rms": 0.7009595279765859,
|
||||
"output_rms": 0.7042018782729917
|
||||
},
|
||||
"k_new_27": {
|
||||
"relative_l2": 0.09483178135454773,
|
||||
"max_abs_error": 0.5629448890686035,
|
||||
"reference_rms": 1.5453936433107067,
|
||||
"output_rms": 1.5464409819108575
|
||||
},
|
||||
"v_new_27": {
|
||||
"relative_l2": 0.09907000401432896,
|
||||
"max_abs_error": 0.26923948526382446,
|
||||
"reference_rms": 0.6731545154533489,
|
||||
"output_rms": 0.6817457220531375
|
||||
}
|
||||
},
|
||||
"cache_max_relative_l2": 0.11714982511905245,
|
||||
"limits": [
|
||||
"The historical source export does not attest an exact checkpoint revision.",
|
||||
"The fixture is synthetic and has no token-to-audio quality tolerance.",
|
||||
"Requested NPU is not actual placement without a device profile.",
|
||||
"No predictor, vocoder, persistent cache loop or complete TTS stream was tested."
|
||||
]
|
||||
}
|
||||
BIN
Binary file not shown.
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:40:20.251995+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "jglyx1v25",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jglyx1v25/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"options": "--compute_unit npu",
|
||||
"hub_version": "aihub-2026.09.11.0",
|
||||
"device": {
|
||||
"name": "Samsung Galaxy S25",
|
||||
"os": "15",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"abi:aarch64-android",
|
||||
"vendor:samsung",
|
||||
"format:phone",
|
||||
"chipset:qualcomm-snapdragon-8-elite-for-galaxy",
|
||||
"chipset:sm8750-ac",
|
||||
"hexagon:v79",
|
||||
"soc-model:69",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"model_id": "mqv3y4xlm",
|
||||
"input_dataset_id": "d7dw6pgy2",
|
||||
"outputs": "device_output.npz"
|
||||
}
|
||||
+400
@@ -0,0 +1,400 @@
|
||||
{
|
||||
"scope": "one synthetic fixed-cache 28-layer Qwen3-TTS talker decode step; no complete TTS stream",
|
||||
"status": "component_executed_numeric_measured",
|
||||
"device": {
|
||||
"name": "SA8775P ADP",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-android",
|
||||
"format:auto",
|
||||
"chipset:qualcomm-sa8775p",
|
||||
"hexagon:v73",
|
||||
"soc-model:52",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"compile_job_id": "j5743w9l5",
|
||||
"inference_job_id": "jgk249evg",
|
||||
"source_model_id": "mq3xre4rq",
|
||||
"target_model_id": "mqe0krw7n",
|
||||
"input_dataset_id": "d7dw6pgy2",
|
||||
"requested_run_options": "--compute_unit npu",
|
||||
"files": {
|
||||
"source_onnx": {
|
||||
"bytes": 1776377476,
|
||||
"sha256": "41feb08517678e94d9301e992d0df610fc60e7c9faba5600703298c86554fd22"
|
||||
},
|
||||
"fixture": {
|
||||
"bytes": 54591500,
|
||||
"sha256": "0bf538e1b64dfc9737dc4f925d61c6e4ffa5075a5dcd7afb1e85ac3bd2d9afea"
|
||||
},
|
||||
"cpu_reference": {
|
||||
"bytes": 237615,
|
||||
"sha256": "ee0ead70e33c02c7d3d6bcb757c5cf0fcbb9eb6fc75c34abfbe8a4479657d5be"
|
||||
},
|
||||
"device_output": {
|
||||
"bytes": 168627,
|
||||
"sha256": "aaa3aaf7f407354a4189889ff435e1bb36b2d8dc8681048903d9a715ad965c52"
|
||||
}
|
||||
},
|
||||
"tensor_comparison": {
|
||||
"logits": {
|
||||
"relative_l2": 0.094269627504538,
|
||||
"max_abs_error": 1.4821429252624512,
|
||||
"reference_rms": 3.7322664515389006,
|
||||
"output_rms": 3.7519194001276905,
|
||||
"cpu_top1": 80,
|
||||
"device_top1": 1720,
|
||||
"top1_matches": false
|
||||
},
|
||||
"k_new_0": {
|
||||
"relative_l2": 0.0005229868214102621,
|
||||
"max_abs_error": 0.003747701644897461,
|
||||
"reference_rms": 1.1349352034374294,
|
||||
"output_rms": 1.1348869273992395
|
||||
},
|
||||
"v_new_0": {
|
||||
"relative_l2": 0.00036238307740231424,
|
||||
"max_abs_error": 0.00020897388458251953,
|
||||
"reference_rms": 0.1383137764927112,
|
||||
"output_rms": 0.13831444664085416
|
||||
},
|
||||
"k_new_1": {
|
||||
"relative_l2": 0.0015594760972754418,
|
||||
"max_abs_error": 0.009575843811035156,
|
||||
"reference_rms": 1.4773734279001252,
|
||||
"output_rms": 1.477313515937877
|
||||
},
|
||||
"v_new_1": {
|
||||
"relative_l2": 0.0015763937695187531,
|
||||
"max_abs_error": 0.0008470192551612854,
|
||||
"reference_rms": 0.162893856079411,
|
||||
"output_rms": 0.16291409038637633
|
||||
},
|
||||
"k_new_2": {
|
||||
"relative_l2": 0.002830993347773807,
|
||||
"max_abs_error": 0.017119139432907104,
|
||||
"reference_rms": 1.4749251890370425,
|
||||
"output_rms": 1.4749381405136908
|
||||
},
|
||||
"v_new_2": {
|
||||
"relative_l2": 0.0032004243530631524,
|
||||
"max_abs_error": 0.003164425492286682,
|
||||
"reference_rms": 0.2996575863196337,
|
||||
"output_rms": 0.2996048804391427
|
||||
},
|
||||
"k_new_3": {
|
||||
"relative_l2": 0.004825412618470713,
|
||||
"max_abs_error": 0.03628116846084595,
|
||||
"reference_rms": 1.4529848472712377,
|
||||
"output_rms": 1.4528570213481324
|
||||
},
|
||||
"v_new_3": {
|
||||
"relative_l2": 0.004807141130895769,
|
||||
"max_abs_error": 0.005009382963180542,
|
||||
"reference_rms": 0.2917197159228712,
|
||||
"output_rms": 0.29158359455043265
|
||||
},
|
||||
"k_new_4": {
|
||||
"relative_l2": 0.005562005244309144,
|
||||
"max_abs_error": 0.046204447746276855,
|
||||
"reference_rms": 1.5413980317857385,
|
||||
"output_rms": 1.5416165713324752
|
||||
},
|
||||
"v_new_4": {
|
||||
"relative_l2": 0.00700852265017866,
|
||||
"max_abs_error": 0.011578202247619629,
|
||||
"reference_rms": 0.41222657123793377,
|
||||
"output_rms": 0.4115125163069992
|
||||
},
|
||||
"k_new_5": {
|
||||
"relative_l2": 0.0070538729135941804,
|
||||
"max_abs_error": 0.05126297473907471,
|
||||
"reference_rms": 1.6285454176549035,
|
||||
"output_rms": 1.6284792479162324
|
||||
},
|
||||
"v_new_5": {
|
||||
"relative_l2": 0.008663118009896303,
|
||||
"max_abs_error": 0.010519325733184814,
|
||||
"reference_rms": 0.32638470026742417,
|
||||
"output_rms": 0.32610640624079057
|
||||
},
|
||||
"k_new_6": {
|
||||
"relative_l2": 0.009656123183392708,
|
||||
"max_abs_error": 0.09657692909240723,
|
||||
"reference_rms": 1.8000231156167423,
|
||||
"output_rms": 1.800494667093283
|
||||
},
|
||||
"v_new_6": {
|
||||
"relative_l2": 0.011705401039366257,
|
||||
"max_abs_error": 0.012994945049285889,
|
||||
"reference_rms": 0.2629708328253682,
|
||||
"output_rms": 0.2627935322426103
|
||||
},
|
||||
"k_new_7": {
|
||||
"relative_l2": 0.01180641301180367,
|
||||
"max_abs_error": 0.1976642608642578,
|
||||
"reference_rms": 1.6255722851184644,
|
||||
"output_rms": 1.6257998066774844
|
||||
},
|
||||
"v_new_7": {
|
||||
"relative_l2": 0.01233106959961381,
|
||||
"max_abs_error": 0.018882550299167633,
|
||||
"reference_rms": 0.348368074126628,
|
||||
"output_rms": 0.34842234046529313
|
||||
},
|
||||
"k_new_8": {
|
||||
"relative_l2": 0.013271891322023667,
|
||||
"max_abs_error": 0.08502709865570068,
|
||||
"reference_rms": 1.516512353065435,
|
||||
"output_rms": 1.5161967827525626
|
||||
},
|
||||
"v_new_8": {
|
||||
"relative_l2": 0.013987967743270447,
|
||||
"max_abs_error": 0.015373915433883667,
|
||||
"reference_rms": 0.28027916002007797,
|
||||
"output_rms": 0.28062639359289654
|
||||
},
|
||||
"k_new_9": {
|
||||
"relative_l2": 0.01572276990977128,
|
||||
"max_abs_error": 0.09883713722229004,
|
||||
"reference_rms": 1.5960043895576705,
|
||||
"output_rms": 1.5968624013575794
|
||||
},
|
||||
"v_new_9": {
|
||||
"relative_l2": 0.020288758436231284,
|
||||
"max_abs_error": 0.024931907653808594,
|
||||
"reference_rms": 0.37541008682864946,
|
||||
"output_rms": 0.37510693299246106
|
||||
},
|
||||
"k_new_10": {
|
||||
"relative_l2": 0.02049303949824089,
|
||||
"max_abs_error": 0.1408187747001648,
|
||||
"reference_rms": 1.6561161468173449,
|
||||
"output_rms": 1.655452207175793
|
||||
},
|
||||
"v_new_10": {
|
||||
"relative_l2": 0.02831931451331964,
|
||||
"max_abs_error": 0.030460327863693237,
|
||||
"reference_rms": 0.26429778504066537,
|
||||
"output_rms": 0.26470174835673915
|
||||
},
|
||||
"k_new_11": {
|
||||
"relative_l2": 0.024059357708676815,
|
||||
"max_abs_error": 0.13748490810394287,
|
||||
"reference_rms": 1.5170462775366995,
|
||||
"output_rms": 1.516467014152262
|
||||
},
|
||||
"v_new_11": {
|
||||
"relative_l2": 0.033134104466359594,
|
||||
"max_abs_error": 0.028084516525268555,
|
||||
"reference_rms": 0.22548908846990579,
|
||||
"output_rms": 0.2261453301417635
|
||||
},
|
||||
"k_new_12": {
|
||||
"relative_l2": 0.02392030350940359,
|
||||
"max_abs_error": 0.13500642776489258,
|
||||
"reference_rms": 1.5731131793440647,
|
||||
"output_rms": 1.5728791562994904
|
||||
},
|
||||
"v_new_12": {
|
||||
"relative_l2": 0.031422763779025996,
|
||||
"max_abs_error": 0.02250637114048004,
|
||||
"reference_rms": 0.20839677682266325,
|
||||
"output_rms": 0.20901102862573526
|
||||
},
|
||||
"k_new_13": {
|
||||
"relative_l2": 0.026838975975458923,
|
||||
"max_abs_error": 0.16691207885742188,
|
||||
"reference_rms": 1.4939452330557514,
|
||||
"output_rms": 1.492831223589065
|
||||
},
|
||||
"v_new_13": {
|
||||
"relative_l2": 0.0349825400340725,
|
||||
"max_abs_error": 0.027702219784259796,
|
||||
"reference_rms": 0.24929462665621008,
|
||||
"output_rms": 0.24977193490279345
|
||||
},
|
||||
"k_new_14": {
|
||||
"relative_l2": 0.02834593856924961,
|
||||
"max_abs_error": 0.17867359519004822,
|
||||
"reference_rms": 1.4942429579997731,
|
||||
"output_rms": 1.4939178835316966
|
||||
},
|
||||
"v_new_14": {
|
||||
"relative_l2": 0.04238903790550551,
|
||||
"max_abs_error": 0.050785861909389496,
|
||||
"reference_rms": 0.32824019866053383,
|
||||
"output_rms": 0.32999641747062985
|
||||
},
|
||||
"k_new_15": {
|
||||
"relative_l2": 0.03397922912988836,
|
||||
"max_abs_error": 0.19253253936767578,
|
||||
"reference_rms": 1.6561072116211841,
|
||||
"output_rms": 1.65513362539146
|
||||
},
|
||||
"v_new_15": {
|
||||
"relative_l2": 0.04930278792969914,
|
||||
"max_abs_error": 0.03966951370239258,
|
||||
"reference_rms": 0.24976283574047556,
|
||||
"output_rms": 0.25285409767917716
|
||||
},
|
||||
"k_new_16": {
|
||||
"relative_l2": 0.04440621232280976,
|
||||
"max_abs_error": 0.29050278663635254,
|
||||
"reference_rms": 1.617983543941328,
|
||||
"output_rms": 1.6163580719201498
|
||||
},
|
||||
"v_new_16": {
|
||||
"relative_l2": 0.05857342016123436,
|
||||
"max_abs_error": 0.08944942057132721,
|
||||
"reference_rms": 0.3826095020438836,
|
||||
"output_rms": 0.3863367712775312
|
||||
},
|
||||
"k_new_17": {
|
||||
"relative_l2": 0.059269240756861116,
|
||||
"max_abs_error": 0.4916331022977829,
|
||||
"reference_rms": 1.6828817809060599,
|
||||
"output_rms": 1.6836984324942421
|
||||
},
|
||||
"v_new_17": {
|
||||
"relative_l2": 0.06312134890689394,
|
||||
"max_abs_error": 0.10612651705741882,
|
||||
"reference_rms": 0.4656876133547257,
|
||||
"output_rms": 0.46725578907411586
|
||||
},
|
||||
"k_new_18": {
|
||||
"relative_l2": 0.06261352395684959,
|
||||
"max_abs_error": 0.40867626667022705,
|
||||
"reference_rms": 1.5546463592216835,
|
||||
"output_rms": 1.5532264621972496
|
||||
},
|
||||
"v_new_18": {
|
||||
"relative_l2": 0.07274058876867781,
|
||||
"max_abs_error": 0.12254154682159424,
|
||||
"reference_rms": 0.5200020604619255,
|
||||
"output_rms": 0.5220507060366361
|
||||
},
|
||||
"k_new_19": {
|
||||
"relative_l2": 0.06585252884093658,
|
||||
"max_abs_error": 0.38734617829322815,
|
||||
"reference_rms": 1.5217070844504548,
|
||||
"output_rms": 1.52036231056702
|
||||
},
|
||||
"v_new_19": {
|
||||
"relative_l2": 0.0784751699988812,
|
||||
"max_abs_error": 0.1748962663114071,
|
||||
"reference_rms": 0.5402920798790236,
|
||||
"output_rms": 0.5438809108946105
|
||||
},
|
||||
"k_new_20": {
|
||||
"relative_l2": 0.0805556587888589,
|
||||
"max_abs_error": 0.717505931854248,
|
||||
"reference_rms": 1.6813854859968058,
|
||||
"output_rms": 1.681003977013983
|
||||
},
|
||||
"v_new_20": {
|
||||
"relative_l2": 0.09078044551764863,
|
||||
"max_abs_error": 0.17253994941711426,
|
||||
"reference_rms": 0.6088730895277438,
|
||||
"output_rms": 0.6091299972846631
|
||||
},
|
||||
"k_new_21": {
|
||||
"relative_l2": 0.07581227648463958,
|
||||
"max_abs_error": 0.6820797324180603,
|
||||
"reference_rms": 1.7370493419005983,
|
||||
"output_rms": 1.7328330204473026
|
||||
},
|
||||
"v_new_21": {
|
||||
"relative_l2": 0.09560749930320718,
|
||||
"max_abs_error": 0.2430351972579956,
|
||||
"reference_rms": 0.6072011016139925,
|
||||
"output_rms": 0.6069685521280092
|
||||
},
|
||||
"k_new_22": {
|
||||
"relative_l2": 0.09170427434779706,
|
||||
"max_abs_error": 1.0033259391784668,
|
||||
"reference_rms": 1.9661056736696203,
|
||||
"output_rms": 1.9638516124645935
|
||||
},
|
||||
"v_new_22": {
|
||||
"relative_l2": 0.10303117868334696,
|
||||
"max_abs_error": 0.2629580646753311,
|
||||
"reference_rms": 0.7362191931025958,
|
||||
"output_rms": 0.7396157791122684
|
||||
},
|
||||
"k_new_23": {
|
||||
"relative_l2": 0.09246479009929692,
|
||||
"max_abs_error": 0.5846302509307861,
|
||||
"reference_rms": 1.6005997999291703,
|
||||
"output_rms": 1.5967480722939222
|
||||
},
|
||||
"v_new_23": {
|
||||
"relative_l2": 0.11335037825257839,
|
||||
"max_abs_error": 0.29803813993930817,
|
||||
"reference_rms": 0.6675198669572269,
|
||||
"output_rms": 0.6577561878631006
|
||||
},
|
||||
"k_new_24": {
|
||||
"relative_l2": 0.09252945712558495,
|
||||
"max_abs_error": 0.7005016803741455,
|
||||
"reference_rms": 1.6337684549178757,
|
||||
"output_rms": 1.642157207382148
|
||||
},
|
||||
"v_new_24": {
|
||||
"relative_l2": 0.114408803401426,
|
||||
"max_abs_error": 0.22830992937088013,
|
||||
"reference_rms": 0.6240472225015637,
|
||||
"output_rms": 0.6256012622599761
|
||||
},
|
||||
"k_new_25": {
|
||||
"relative_l2": 0.08126826488095878,
|
||||
"max_abs_error": 0.44124507904052734,
|
||||
"reference_rms": 1.527601662385539,
|
||||
"output_rms": 1.5316483699485346
|
||||
},
|
||||
"v_new_25": {
|
||||
"relative_l2": 0.10256024595914798,
|
||||
"max_abs_error": 0.3586934506893158,
|
||||
"reference_rms": 0.9133071626950193,
|
||||
"output_rms": 0.9099123411818925
|
||||
},
|
||||
"k_new_26": {
|
||||
"relative_l2": 0.07581284164196181,
|
||||
"max_abs_error": 0.4462745040655136,
|
||||
"reference_rms": 1.4905787414732559,
|
||||
"output_rms": 1.4935160005691213
|
||||
},
|
||||
"v_new_26": {
|
||||
"relative_l2": 0.11537698355642578,
|
||||
"max_abs_error": 0.25528641790151596,
|
||||
"reference_rms": 0.7009595279765859,
|
||||
"output_rms": 0.704761171210935
|
||||
},
|
||||
"k_new_27": {
|
||||
"relative_l2": 0.09246421199602507,
|
||||
"max_abs_error": 0.5224957466125488,
|
||||
"reference_rms": 1.5453936433107067,
|
||||
"output_rms": 1.5463617281369817
|
||||
},
|
||||
"v_new_27": {
|
||||
"relative_l2": 0.09868798043878668,
|
||||
"max_abs_error": 0.20730507373809814,
|
||||
"reference_rms": 0.6731545154533489,
|
||||
"output_rms": 0.6811154098942891
|
||||
}
|
||||
},
|
||||
"cache_max_relative_l2": 0.11537698355642578,
|
||||
"limits": [
|
||||
"The historical source export does not attest an exact checkpoint revision.",
|
||||
"The fixture is synthetic and has no token-to-audio quality tolerance.",
|
||||
"Requested NPU is not actual placement without a device profile.",
|
||||
"No predictor, vocoder, persistent cache loop or complete TTS stream was tested."
|
||||
]
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:47:57.336406+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_model_id": "mq3xre4rq",
|
||||
"device": "SA8775P ADP",
|
||||
"options": "--target_runtime tflite",
|
||||
"kind": "compile",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jgolr4wqg",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jgolr4wqg/"
|
||||
}
|
||||
BIN
Binary file not shown.
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:38:58.253843+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "jgk249evg",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jgk249evg/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"options": "--compute_unit npu",
|
||||
"hub_version": "aihub-2026.09.11.0",
|
||||
"device": {
|
||||
"name": "SA8775P ADP",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-android",
|
||||
"format:auto",
|
||||
"chipset:qualcomm-sa8775p",
|
||||
"hexagon:v73",
|
||||
"soc-model:52",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"model_id": "mqe0krw7n",
|
||||
"input_dataset_id": "d7dw6pgy2",
|
||||
"outputs": "device_output.npz"
|
||||
}
|
||||
+400
@@ -0,0 +1,400 @@
|
||||
{
|
||||
"scope": "one synthetic fixed-cache 28-layer Qwen3-TTS talker decode step; no complete TTS stream",
|
||||
"status": "component_executed_numeric_measured",
|
||||
"device": {
|
||||
"name": "Snapdragon X Elite CRD",
|
||||
"os": "11",
|
||||
"attributes": [
|
||||
"os:windows",
|
||||
"format:compute",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-windows",
|
||||
"chipset:qualcomm-snapdragon-x-elite",
|
||||
"chipset:sc8380xp",
|
||||
"hexagon:v73",
|
||||
"soc-model:60",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"compile_job_id": "jp1jxl62p",
|
||||
"inference_job_id": "j5qlym6ep",
|
||||
"source_model_id": "mq3xre4rq",
|
||||
"target_model_id": "mqv3y1g0m",
|
||||
"input_dataset_id": "d7dw6pgy2",
|
||||
"requested_run_options": "--compute_unit npu",
|
||||
"files": {
|
||||
"source_onnx": {
|
||||
"bytes": 1776377476,
|
||||
"sha256": "41feb08517678e94d9301e992d0df610fc60e7c9faba5600703298c86554fd22"
|
||||
},
|
||||
"fixture": {
|
||||
"bytes": 54591500,
|
||||
"sha256": "0bf538e1b64dfc9737dc4f925d61c6e4ffa5075a5dcd7afb1e85ac3bd2d9afea"
|
||||
},
|
||||
"cpu_reference": {
|
||||
"bytes": 237615,
|
||||
"sha256": "ee0ead70e33c02c7d3d6bcb757c5cf0fcbb9eb6fc75c34abfbe8a4479657d5be"
|
||||
},
|
||||
"device_output": {
|
||||
"bytes": 168583,
|
||||
"sha256": "27cb1f8d39ba15c7149df556251e9b196afb8cd2e4ca6e2d38ea6b3366cb6e61"
|
||||
}
|
||||
},
|
||||
"tensor_comparison": {
|
||||
"logits": {
|
||||
"relative_l2": 0.08571460504486464,
|
||||
"max_abs_error": 1.370536983013153,
|
||||
"reference_rms": 3.7322664515389006,
|
||||
"output_rms": 3.7421927052928883,
|
||||
"cpu_top1": 80,
|
||||
"device_top1": 1720,
|
||||
"top1_matches": false
|
||||
},
|
||||
"k_new_0": {
|
||||
"relative_l2": 0.0005229822431162853,
|
||||
"max_abs_error": 0.003747701644897461,
|
||||
"reference_rms": 1.1349352034374294,
|
||||
"output_rms": 1.1348869276631997
|
||||
},
|
||||
"v_new_0": {
|
||||
"relative_l2": 0.0003622125877911652,
|
||||
"max_abs_error": 0.00020897388458251953,
|
||||
"reference_rms": 0.1383137764927112,
|
||||
"output_rms": 0.1383144487129386
|
||||
},
|
||||
"k_new_1": {
|
||||
"relative_l2": 0.0015669701135229024,
|
||||
"max_abs_error": 0.008643865585327148,
|
||||
"reference_rms": 1.4773734279001252,
|
||||
"output_rms": 1.4773104889841304
|
||||
},
|
||||
"v_new_1": {
|
||||
"relative_l2": 0.0015795827355481907,
|
||||
"max_abs_error": 0.0008470192551612854,
|
||||
"reference_rms": 0.162893856079411,
|
||||
"output_rms": 0.16291260371204075
|
||||
},
|
||||
"k_new_2": {
|
||||
"relative_l2": 0.0027858818202019906,
|
||||
"max_abs_error": 0.016142576932907104,
|
||||
"reference_rms": 1.4749251890370425,
|
||||
"output_rms": 1.4748933477302866
|
||||
},
|
||||
"v_new_2": {
|
||||
"relative_l2": 0.0030928222765495336,
|
||||
"max_abs_error": 0.0031082630157470703,
|
||||
"reference_rms": 0.2996575863196337,
|
||||
"output_rms": 0.2995644188746399
|
||||
},
|
||||
"k_new_3": {
|
||||
"relative_l2": 0.004827601205092268,
|
||||
"max_abs_error": 0.03335148096084595,
|
||||
"reference_rms": 1.4529848472712377,
|
||||
"output_rms": 1.4528312527620655
|
||||
},
|
||||
"v_new_3": {
|
||||
"relative_l2": 0.004769322977576131,
|
||||
"max_abs_error": 0.005497664213180542,
|
||||
"reference_rms": 0.2917197159228712,
|
||||
"output_rms": 0.2915401628551229
|
||||
},
|
||||
"k_new_4": {
|
||||
"relative_l2": 0.005367998145569408,
|
||||
"max_abs_error": 0.045227885246276855,
|
||||
"reference_rms": 1.5413980317857385,
|
||||
"output_rms": 1.5416631164696781
|
||||
},
|
||||
"v_new_4": {
|
||||
"relative_l2": 0.0072768824742693635,
|
||||
"max_abs_error": 0.013531327247619629,
|
||||
"reference_rms": 0.41222657123793377,
|
||||
"output_rms": 0.4112383786590707
|
||||
},
|
||||
"k_new_5": {
|
||||
"relative_l2": 0.006765111772419511,
|
||||
"max_abs_error": 0.05028641223907471,
|
||||
"reference_rms": 1.6285454176549035,
|
||||
"output_rms": 1.6285332163572785
|
||||
},
|
||||
"v_new_5": {
|
||||
"relative_l2": 0.00855806698559772,
|
||||
"max_abs_error": 0.011495888233184814,
|
||||
"reference_rms": 0.32638470026742417,
|
||||
"output_rms": 0.32595237474174477
|
||||
},
|
||||
"k_new_6": {
|
||||
"relative_l2": 0.009522328352257983,
|
||||
"max_abs_error": 0.09950661659240723,
|
||||
"reference_rms": 1.8000231156167423,
|
||||
"output_rms": 1.800524902063639
|
||||
},
|
||||
"v_new_6": {
|
||||
"relative_l2": 0.011765931057437985,
|
||||
"max_abs_error": 0.011530101299285889,
|
||||
"reference_rms": 0.2629708328253682,
|
||||
"output_rms": 0.2626820793624253
|
||||
},
|
||||
"k_new_7": {
|
||||
"relative_l2": 0.01122956925104222,
|
||||
"max_abs_error": 0.1761798858642578,
|
||||
"reference_rms": 1.6255722851184644,
|
||||
"output_rms": 1.6258440040497664
|
||||
},
|
||||
"v_new_7": {
|
||||
"relative_l2": 0.012161154327846287,
|
||||
"max_abs_error": 0.018272198736667633,
|
||||
"reference_rms": 0.348368074126628,
|
||||
"output_rms": 0.34820183690017315
|
||||
},
|
||||
"k_new_8": {
|
||||
"relative_l2": 0.013354808880270798,
|
||||
"max_abs_error": 0.081199049949646,
|
||||
"reference_rms": 1.516512353065435,
|
||||
"output_rms": 1.5163190004916662
|
||||
},
|
||||
"v_new_8": {
|
||||
"relative_l2": 0.013811113070532429,
|
||||
"max_abs_error": 0.016350477933883667,
|
||||
"reference_rms": 0.28027916002007797,
|
||||
"output_rms": 0.2804339388817987
|
||||
},
|
||||
"k_new_9": {
|
||||
"relative_l2": 0.015884529017628282,
|
||||
"max_abs_error": 0.09493088722229004,
|
||||
"reference_rms": 1.5960043895576705,
|
||||
"output_rms": 1.596565781150863
|
||||
},
|
||||
"v_new_9": {
|
||||
"relative_l2": 0.019960363235920445,
|
||||
"max_abs_error": 0.024931907653808594,
|
||||
"reference_rms": 0.37541008682864946,
|
||||
"output_rms": 0.374677147921929
|
||||
},
|
||||
"k_new_10": {
|
||||
"relative_l2": 0.020332897518330582,
|
||||
"max_abs_error": 0.1417953372001648,
|
||||
"reference_rms": 1.6561161468173449,
|
||||
"output_rms": 1.6555391321179171
|
||||
},
|
||||
"v_new_10": {
|
||||
"relative_l2": 0.028332755912675384,
|
||||
"max_abs_error": 0.03314587473869324,
|
||||
"reference_rms": 0.26429778504066537,
|
||||
"output_rms": 0.2644657528914953
|
||||
},
|
||||
"k_new_11": {
|
||||
"relative_l2": 0.02386321408307146,
|
||||
"max_abs_error": 0.13846147060394287,
|
||||
"reference_rms": 1.5170462775366995,
|
||||
"output_rms": 1.5165213264128052
|
||||
},
|
||||
"v_new_11": {
|
||||
"relative_l2": 0.03286288336342349,
|
||||
"max_abs_error": 0.026897940784692764,
|
||||
"reference_rms": 0.22548908846990579,
|
||||
"output_rms": 0.22590644526351483
|
||||
},
|
||||
"k_new_12": {
|
||||
"relative_l2": 0.024049316967754742,
|
||||
"max_abs_error": 0.13891267776489258,
|
||||
"reference_rms": 1.5731131793440647,
|
||||
"output_rms": 1.573051503394554
|
||||
},
|
||||
"v_new_12": {
|
||||
"relative_l2": 0.03141548933568732,
|
||||
"max_abs_error": 0.023222461342811584,
|
||||
"reference_rms": 0.20839677682266325,
|
||||
"output_rms": 0.20882429085804227
|
||||
},
|
||||
"k_new_13": {
|
||||
"relative_l2": 0.02722860732949299,
|
||||
"max_abs_error": 0.17863082885742188,
|
||||
"reference_rms": 1.4939452330557514,
|
||||
"output_rms": 1.4928709975786807
|
||||
},
|
||||
"v_new_13": {
|
||||
"relative_l2": 0.03525129996186022,
|
||||
"max_abs_error": 0.028617747128009796,
|
||||
"reference_rms": 0.24929462665621008,
|
||||
"output_rms": 0.24947246830365316
|
||||
},
|
||||
"k_new_14": {
|
||||
"relative_l2": 0.02841141105798686,
|
||||
"max_abs_error": 0.17330250144004822,
|
||||
"reference_rms": 1.4942429579997731,
|
||||
"output_rms": 1.4941630177959875
|
||||
},
|
||||
"v_new_14": {
|
||||
"relative_l2": 0.042532815366008975,
|
||||
"max_abs_error": 0.05243998393416405,
|
||||
"reference_rms": 0.32824019866053383,
|
||||
"output_rms": 0.32938043730972877
|
||||
},
|
||||
"k_new_15": {
|
||||
"relative_l2": 0.03320074206878956,
|
||||
"max_abs_error": 0.20327472686767578,
|
||||
"reference_rms": 1.6561072116211841,
|
||||
"output_rms": 1.6550050412282318
|
||||
},
|
||||
"v_new_15": {
|
||||
"relative_l2": 0.048068133067658406,
|
||||
"max_abs_error": 0.0410936176776886,
|
||||
"reference_rms": 0.24976283574047556,
|
||||
"output_rms": 0.25254640339243767
|
||||
},
|
||||
"k_new_16": {
|
||||
"relative_l2": 0.04347488531139934,
|
||||
"max_abs_error": 0.305935800075531,
|
||||
"reference_rms": 1.617983543941328,
|
||||
"output_rms": 1.6162195104886656
|
||||
},
|
||||
"v_new_16": {
|
||||
"relative_l2": 0.05706519742364159,
|
||||
"max_abs_error": 0.08175899088382721,
|
||||
"reference_rms": 0.3826095020438836,
|
||||
"output_rms": 0.38605486706589076
|
||||
},
|
||||
"k_new_17": {
|
||||
"relative_l2": 0.05692172320041135,
|
||||
"max_abs_error": 0.4672190397977829,
|
||||
"reference_rms": 1.6828817809060599,
|
||||
"output_rms": 1.6836293204378896
|
||||
},
|
||||
"v_new_17": {
|
||||
"relative_l2": 0.06109146685031243,
|
||||
"max_abs_error": 0.09391948580741882,
|
||||
"reference_rms": 0.4656876133547257,
|
||||
"output_rms": 0.467225404804508
|
||||
},
|
||||
"k_new_18": {
|
||||
"relative_l2": 0.06327824206180248,
|
||||
"max_abs_error": 0.4432591199874878,
|
||||
"reference_rms": 1.5546463592216835,
|
||||
"output_rms": 1.5527866306207152
|
||||
},
|
||||
"v_new_18": {
|
||||
"relative_l2": 0.07214879600603967,
|
||||
"max_abs_error": 0.12156498432159424,
|
||||
"reference_rms": 0.5200020604619255,
|
||||
"output_rms": 0.5212353545971804
|
||||
},
|
||||
"k_new_19": {
|
||||
"relative_l2": 0.06602797373144118,
|
||||
"max_abs_error": 0.3827641010284424,
|
||||
"reference_rms": 1.5217070844504548,
|
||||
"output_rms": 1.5202632035044439
|
||||
},
|
||||
"v_new_19": {
|
||||
"relative_l2": 0.07854801677759958,
|
||||
"max_abs_error": 0.1744079850614071,
|
||||
"reference_rms": 0.5402920798790236,
|
||||
"output_rms": 0.5440763397165683
|
||||
},
|
||||
"k_new_20": {
|
||||
"relative_l2": 0.08083411451454492,
|
||||
"max_abs_error": 0.768287181854248,
|
||||
"reference_rms": 1.6813854859968058,
|
||||
"output_rms": 1.681736554042449
|
||||
},
|
||||
"v_new_20": {
|
||||
"relative_l2": 0.09175638094504608,
|
||||
"max_abs_error": 0.18032324314117432,
|
||||
"reference_rms": 0.6088730895277438,
|
||||
"output_rms": 0.6083971065410384
|
||||
},
|
||||
"k_new_21": {
|
||||
"relative_l2": 0.07795297576951524,
|
||||
"max_abs_error": 0.6601070761680603,
|
||||
"reference_rms": 1.7370493419005983,
|
||||
"output_rms": 1.7332221423469922
|
||||
},
|
||||
"v_new_21": {
|
||||
"relative_l2": 0.09523532798374042,
|
||||
"max_abs_error": 0.2235039472579956,
|
||||
"reference_rms": 0.6072011016139925,
|
||||
"output_rms": 0.6066838349539903
|
||||
},
|
||||
"k_new_22": {
|
||||
"relative_l2": 0.0899867558806697,
|
||||
"max_abs_error": 0.9447321891784668,
|
||||
"reference_rms": 1.9661056736696203,
|
||||
"output_rms": 1.962542102594991
|
||||
},
|
||||
"v_new_22": {
|
||||
"relative_l2": 0.09955628631665578,
|
||||
"max_abs_error": 0.24879790842533112,
|
||||
"reference_rms": 0.7362191931025958,
|
||||
"output_rms": 0.7361582531705974
|
||||
},
|
||||
"k_new_23": {
|
||||
"relative_l2": 0.09379468349046649,
|
||||
"max_abs_error": 0.6442005634307861,
|
||||
"reference_rms": 1.6005997999291703,
|
||||
"output_rms": 1.597624770237883
|
||||
},
|
||||
"v_new_23": {
|
||||
"relative_l2": 0.11345453698229609,
|
||||
"max_abs_error": 0.29730571806430817,
|
||||
"reference_rms": 0.6675198669572269,
|
||||
"output_rms": 0.6584833287091308
|
||||
},
|
||||
"k_new_24": {
|
||||
"relative_l2": 0.09262508605228019,
|
||||
"max_abs_error": 0.6484112739562988,
|
||||
"reference_rms": 1.6337684549178757,
|
||||
"output_rms": 1.6424048217094986
|
||||
},
|
||||
"v_new_24": {
|
||||
"relative_l2": 0.11496896932625597,
|
||||
"max_abs_error": 0.23512649536132812,
|
||||
"reference_rms": 0.6240472225015637,
|
||||
"output_rms": 0.6250522430093708
|
||||
},
|
||||
"k_new_25": {
|
||||
"relative_l2": 0.08107521634955106,
|
||||
"max_abs_error": 0.4823617935180664,
|
||||
"reference_rms": 1.527601662385539,
|
||||
"output_rms": 1.5328151290850238
|
||||
},
|
||||
"v_new_25": {
|
||||
"relative_l2": 0.09927459901725227,
|
||||
"max_abs_error": 0.3826192319393158,
|
||||
"reference_rms": 0.9133071626950193,
|
||||
"output_rms": 0.9091111003598573
|
||||
},
|
||||
"k_new_26": {
|
||||
"relative_l2": 0.0733050459139669,
|
||||
"max_abs_error": 0.407822348177433,
|
||||
"reference_rms": 1.4905787414732559,
|
||||
"output_rms": 1.4930591029965392
|
||||
},
|
||||
"v_new_26": {
|
||||
"relative_l2": 0.1127212565374478,
|
||||
"max_abs_error": 0.2557843029499054,
|
||||
"reference_rms": 0.7009595279765859,
|
||||
"output_rms": 0.7041752859333699
|
||||
},
|
||||
"k_new_27": {
|
||||
"relative_l2": 0.0901155189215236,
|
||||
"max_abs_error": 0.5277888774871826,
|
||||
"reference_rms": 1.5453936433107067,
|
||||
"output_rms": 1.547406174191951
|
||||
},
|
||||
"v_new_27": {
|
||||
"relative_l2": 0.09665042151849512,
|
||||
"max_abs_error": 0.24775516986846924,
|
||||
"reference_rms": 0.6731545154533489,
|
||||
"output_rms": 0.6810086528333622
|
||||
}
|
||||
},
|
||||
"cache_max_relative_l2": 0.11496896932625597,
|
||||
"limits": [
|
||||
"The historical source export does not attest an exact checkpoint revision.",
|
||||
"The fixture is synthetic and has no token-to-audio quality tolerance.",
|
||||
"Requested NPU is not actual placement without a device profile.",
|
||||
"No predictor, vocoder, persistent cache loop or complete TTS stream was tested."
|
||||
]
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"retrieved_utc": "2026-09-23T23:50:27.441379+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"job_id": "jp1njv18g",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jp1njv18g/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"source_model_id": "mq3xre4rq",
|
||||
"target_model_id": "mnwv5rp3q",
|
||||
"device": {
|
||||
"name": "Snapdragon X Elite CRD",
|
||||
"os": "11",
|
||||
"attributes": [
|
||||
"os:windows",
|
||||
"format:compute",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-windows",
|
||||
"chipset:qualcomm-snapdragon-x-elite",
|
||||
"chipset:sc8380xp",
|
||||
"hexagon:v73",
|
||||
"soc-model:60",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"options": "--target_runtime onnx",
|
||||
"hub_version": "aihub-2026.09.11.0"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:45:36.274399+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_model_id": "mq3xre4rq",
|
||||
"device": "Snapdragon X Elite CRD",
|
||||
"options": "--target_runtime onnx",
|
||||
"kind": "compile",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jp1njv18g",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jp1njv18g/"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:50:29.213280+00:00",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"source_compile_job_id": "jp1njv18g",
|
||||
"target_model_id": "mnwv5rp3q",
|
||||
"input_dataset_id": "d7dw6pgy2",
|
||||
"device": "Snapdragon X Elite CRD",
|
||||
"options": "--compute_unit cpu",
|
||||
"kind": "inference",
|
||||
"status": "SUBMITTED",
|
||||
"job_id": "jgzlm40o5",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jgzlm40o5/"
|
||||
}
|
||||
BIN
Binary file not shown.
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:42:22.950524+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "j5qlym6ep",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/j5qlym6ep/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"options": "--compute_unit npu",
|
||||
"hub_version": "aihub-2026.09.11.0",
|
||||
"device": {
|
||||
"name": "Snapdragon X Elite CRD",
|
||||
"os": "11",
|
||||
"attributes": [
|
||||
"os:windows",
|
||||
"format:compute",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-windows",
|
||||
"chipset:qualcomm-snapdragon-x-elite",
|
||||
"chipset:sc8380xp",
|
||||
"hexagon:v73",
|
||||
"soc-model:60",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"model_id": "mqv3y1g0m",
|
||||
"input_dataset_id": "d7dw6pgy2",
|
||||
"outputs": "device_output.npz"
|
||||
}
|
||||
+2
-2
@@ -10,12 +10,12 @@ The fresh [RB3 inference](rb3/inference_report.json) on the same fixture returne
|
||||
|
||||
A separate [local LiteRT 2.2.0 CPU control](local_litert_cpu/report.json) ran that **same pinned TFLite binary** on the WSL x86-64 HX370 host with four interpreter threads. Its [waveform](local_litert_cpu/device_output.npz) was finite and unsaturated, within **3.45e-6 relative L2 / 109.23 dB SNR** of the original ONNX Runtime CPU output. The [reproducer](../../../../experiments/probe_qwen_tts_vocoder_litert_cpu.py) checks binary/fixture/reference hashes and I/O contracts. This narrows the SA saturation to hosted runtime/device behavior rather than a universally bad TFLite conversion. The one local invoke is not a comparable performance profile; it does not prove either embedded device's CPU route or identify the specific delegate operation at fault.
|
||||
|
||||
On the **exact hosted SA8775P**, a [CPU-requested inference](sa8775p/cpu_control/inference_report.json) of that binary also returned a finite unsaturated waveform. The [same-fixture comparison](sa8775p/cpu_control/inference_comparison.json) was **4.13e-6 relative L2 / 107.67 dB SNR** against original ONNX CPU. The GPU-requested SA output differed from this SA CPU-requested output by **11.832 relative L2** and saturated 96.63% of samples. This is strong evidence that the tested GPU-requested route causes the waveform failure on SA, while its exact delegate operation and all CPU placement details still require the [CPU profile](sa8775p/cpu_control/profile_submission.json) and FP32-preserving GPU control.
|
||||
On the **exact hosted SA8775P**, a [CPU-requested inference](sa8775p/cpu_control/inference_report.json) of that binary also returned a finite unsaturated waveform. The [same-fixture comparison](sa8775p/cpu_control/inference_comparison.json) was **4.13e-6 relative L2 / 107.67 dB SNR** against original ONNX CPU. The GPU-requested SA output differed from this SA CPU-requested output by **11.832 relative L2** and saturated 96.63% of samples. A fresh [FP32-preserving GPU-option inference](sa8775p/gpu_fp32/inference_report.json) also failed numerically: its [comparison](sa8775p/gpu_fp32/inference_comparison.json) had **11.389 relative L2 / −21.13 dB SNR** versus original ONNX CPU and **84.89%** saturated samples. Disabling the FP32-to-FP16 allowance does not rescue this tested GPU route. The exact faulty delegate operation and all CPU placement details still require further isolation and the [CPU profile](sa8775p/cpu_control/profile_submission.json).
|
||||
|
||||
The re-fetched *historical* [SA8775P profile](sa8775p/historical_profile_report.json) has 100 component samples, nearest-rank p50/p95 **1.751/1.874 s**, and 403 CPU plus 335 GPU execution-detail rows. The *historical* [RB3 profile](rb3/historical_profile_report.json) has 100 samples, p50/p95 **3.143/3.189 s**, also with 403 CPU plus 335 GPU rows. Their previously quoted 1.701/3.107 s estimated-time fields are sample minima, not medians. Both profiles exclude talker/predictor, stage handoff, streaming and full-request time.
|
||||
|
||||
The X Elite [ONNX CPU inference](xelite/cpu/inference_report.json) succeeded on the pinned fixture. Its [waveform comparison](xelite/cpu/inference_comparison.json) to the original local ONNX Runtime CPU output was **8.09e-7 relative L2 / 121.84 dB SNR**, with no saturation. The fresh [CPU profile](xelite/cpu/profile_report.json) returned **94** component samples, nearest-rank p50/p95 **6.384/6.644 s**, with all **634** execution-detail rows on CPU; the service's 5.917 s estimated-time field is the sample minimum. The [full audit](xelite/cpu/audit_report.json) pins source, fixture, target, waveform, placement and percentiles. This is near-source component parity on one fixture, without a listening-quality or complete-stream gate. The first X Elite [GPU inference](xelite/gpu/inference_report.json) and [profile](xelite/gpu/profile_report.json) jobs failed because Workbench rejects specifying both compute unit and ONNX execution provider. The corrected [DirectML inference](xelite/directml/inference_report.json) and [profile](xelite/directml/profile_report.json) both failed during DML provider initialization with Windows error `80070057` (`The parameter is incorrect`), without an output or timing samples. A separate [FP16 QNN DLC compile](xelite/qnn_npu/compile_report.json) succeeded and [NPU-requested same-fixture inference](xelite/qnn_npu/inference_submission.json) was pending. The [X Elite ONNX CPU compile](xelite/compile_report.json) passed after the older TFLite target was rejected on Windows ARM. These failures constrain the tested DirectML route, not all GPU or NPU artifacts.
|
||||
|
||||
The [SA CPU profile](sa8775p/cpu_control/profile_submission.json), [FP32-preserving GPU](sa8775p/gpu_fp32/inference_submission.json) control, and X Elite [QNN NPU inference](xelite/qnn_npu/inference_submission.json) remained pending in this snapshot. Pending jobs have no numerical or latency conclusion. The [collector](../../../../experiments/collect_aihub_component.py) retrieves their terminal reports without resubmitting jobs, and the [audit script](../../../../experiments/audit_qwen_tts_vocoder_qualcomm.py) checks source/fixture hashes, device and model IDs, actual placement, waveform error, saturation and sample percentiles.
|
||||
The [SA CPU profile](sa8775p/cpu_control/profile_submission.json) and X Elite [QNN NPU inference](xelite/qnn_npu/inference_submission.json) remained pending in this snapshot. Pending jobs have no numerical or latency conclusion. The [collector](../../../../experiments/collect_aihub_component.py) retrieves their terminal reports without resubmitting jobs, and the [audit script](../../../../experiments/audit_qwen_tts_vocoder_qualcomm.py) checks source/fixture hashes, device and model IDs, actual placement, waveform error, saturation and sample percentiles.
|
||||
|
||||
This is one fixed vocoder component with a retained input, not complete Qwen3-TTS. The next gates are a quality-passing SA/RB3/X Elite target, talker and code predictor numerical checks, device-resident handoff/co-residency, playable streaming, cancellation, memory admission and sustained power/thermal profiling.
|
||||
BIN
Binary file not shown.
+51
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"scope": "one Qwen3-TTS Code2Wav inference with FP32-preserving GPU option on SA8775P; no complete TTS stream",
|
||||
"status": "component_gpu_requested_numeric_fail",
|
||||
"compile_job_id": "jp410o3vp",
|
||||
"inference_job_id": "jgdd38lzg",
|
||||
"target_model_id": "mnze95don",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"requested_options": "--compute_unit gpu --tflite_options allow_fp32_as_fp16=false",
|
||||
"device": {
|
||||
"name": "SA8775P ADP",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-android",
|
||||
"format:auto",
|
||||
"chipset:qualcomm-sa8775p",
|
||||
"hexagon:v73",
|
||||
"soc-model:52",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"output_sha256": "9196d254564a1286f56da5d415f69cb0b85d6dbbc5550f6d31b818ab2eab5b42",
|
||||
"source_onnx_cpu_vs_sa_gpu_fp32": {
|
||||
"relative_l2": 11.389093101414934,
|
||||
"snr_db": -21.129782863088515,
|
||||
"max_abs": 1.4882186353206635,
|
||||
"cosine": -0.018527488603516624,
|
||||
"reference_rms": 0.08373439205485074,
|
||||
"output_rms": 0.9484254750023428,
|
||||
"output_saturated_fraction_abs_ge_0_999": 0.8488958333333333
|
||||
},
|
||||
"sa_cpu_requested_vs_sa_gpu_fp32": {
|
||||
"relative_l2": 11.389092347977442,
|
||||
"snr_db": -21.12978228847954,
|
||||
"max_abs": 1.4882185459136963,
|
||||
"cosine": -0.01852748991918297,
|
||||
"reference_rms": 0.08373439765610562,
|
||||
"output_rms": 0.9484254750023428,
|
||||
"output_saturated_fraction_abs_ge_0_999": 0.8488958333333333
|
||||
},
|
||||
"limits": [
|
||||
"No actual per-node placement profile for this control.",
|
||||
"The output is severely saturated, so component timing would not be a usable TTS measure.",
|
||||
"The option result does not identify a specific faulty GPU operation."
|
||||
]
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"timestamp_utc": "2026-09-23T23:38:58.225134+00:00",
|
||||
"qai_hub_version": "0.55.0",
|
||||
"facility": "Qualcomm AI Hub Workbench",
|
||||
"job_id": "jgdd38lzg",
|
||||
"url": "https://workbench.aihub.qualcomm.com/jobs/jgdd38lzg/",
|
||||
"status": "SUCCESS",
|
||||
"status_message": "",
|
||||
"options": "--compute_unit gpu --tflite_options allow_fp32_as_fp16=false",
|
||||
"hub_version": "aihub-2026.09.11.0",
|
||||
"device": {
|
||||
"name": "SA8775P ADP",
|
||||
"os": "14",
|
||||
"attributes": [
|
||||
"os:android",
|
||||
"framework:tflite",
|
||||
"framework:onnx",
|
||||
"vendor:qualcomm",
|
||||
"abi:aarch64-android",
|
||||
"format:auto",
|
||||
"chipset:qualcomm-sa8775p",
|
||||
"hexagon:v73",
|
||||
"soc-model:52",
|
||||
"htp-supports-fp16:true",
|
||||
"htp-supports-weight-sharing:true",
|
||||
"framework:qnn"
|
||||
]
|
||||
},
|
||||
"model_id": "mnze95don",
|
||||
"input_dataset_id": "d7m80jl32",
|
||||
"outputs": "device_output.npz"
|
||||
}
|
||||
@@ -16,9 +16,11 @@ A [Qwen3-TTS Code2Wav follow-up](../../../e2e_expansion_20260924/evidence/qwen_t
|
||||
|
||||
The identical SA/RB3 TFLite binary matched original ONNX CPU within 3.45e-6 relative L2 in a separate local HX370 x86-64 LiteRT CPU control. This narrows the SA waveform saturation to hosted device/runtime behavior, without proving either embedded CPU route or a specific GPU-delegate fault.
|
||||
|
||||
A separate [Qwen3-TTS talker decode-step attempt](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) recovered a 28-layer ONNX export and synthetic fixed-cache fixture. Historical FP16 QNN context-binary compiles passed for S24, S25, SA8775P and X Elite, but the same-fixture inference jobs were pending. An RB3 compile rejected floating-point input. No talker runtime or numerical pass is implied by those compile outcomes.
|
||||
A separate [Qwen3-TTS talker decode-step attempt](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) recovered a 28-layer ONNX export and synthetic fixed-cache fixture. Historical FP16 QNN context-binary inference executed on S24, S25, SA8775P and X Elite, but all four changed the CPU reference top-1 speech token with 8.57–9.43% logits relative L2; they are numerically unqualified on this fixture. An RB3 compile rejected floating-point input. No complete talker loop or TTS stream follows.
|
||||
|
||||
A [Galaxy S25 InternVLA Cosmos attempt](../../../e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md) packaged the real Place_Markpen image encoder into one ONNX file after two CPU parity fixtures. FLOAT16 QNN DLC compilation passed, while same-fixture NPU-requested inference was pending. This changes the artifact investigation for that cell, not its NPU or full-policy qualification.
|
||||
A [Galaxy S25 InternVLA Cosmos attempt](../../../e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md) packaged the real Place_Markpen image encoder into one ONNX file after two CPU parity fixtures. FLOAT16 QNN DLC inference returned a latent within 0.0524% relative L2 of the source on one pattern; device placement profile was pending. Injecting that latent into the real policy on local CPU changed synthetic actions by 0.742% relative L2, without a task tolerance. This advances component and action-sensitivity evidence, not device-local full-policy qualification.
|
||||
|
||||
A [MiniCPM-o speech-head study](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md) recovered the historical 20-layer source and matching S24/S25 fixtures. Both FP16 context-binary outputs preserved the ONNX CPU top-1 speech token on this one decode step with logits relative L2 of 0.346%/0.360%. New FP16 QNN DLC compiles passed for SA8775P, X Elite and RB3; same-fixture NPU-requested inference was pending. These are speech-head component results, not multimodal MiniCPM-o requests.
|
||||
|
||||
## Configuration × model
|
||||
|
||||
@@ -85,29 +87,29 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | InternVLA-A1 | NOT_E2E_PROFILEABLE | No qualifying same-model multi-accelerator deployment/artifact plan or overlap/recovery evidence. Individual routes are not a joint pipeline. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Spark-X2.5 | COMPONENT_NPU_INFERENCE_PROFILED; NOT_E2E_PROFILEABLE | A real Spark-X2.5-1.7B one-layer W8A16 QDQ attention artifact, compiled as QNN DLC, ran on the exact hosted Snapdragon X Elite CRD (Windows 11 ARM64). Inference returned finite hidden/new-K/new-V tensors bitwise identical to the prior S24/S25 NPU outputs on the same fixture; the profile attributed all 73 execution-detail rows to NPU. Across 100 component samples, nearest-rank p50/p95 was 1.433/1.606 ms; the 1.346 ms estimated-time field is the minimum. Hidden/K/V relative L2 versus original FP32 ONNX CPU was 1.351%/0.377%/0.650%, without a next-token quality tolerance. There is no complete 28-layer prefill/decode/KV-ring/sampling loop, resident Omni controller or complete-request timing. | Qualify a complete X Elite resident artifact/controller with state, token quality, memory admission/cancel and sustained power; [X Elite NPU raw evidence](../../../e2e_expansion_20260923/evidence/spark_xelite_full_attention/README.md). |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_ONNX_CPU_VOCODER_PARITY_PROFILED; DIRECTML_RUNTIME_FAILED; QNN_NPU_COMPILE_PASS; NOT_E2E_PROFILEABLE | The fresh ONNX vocoder inference on the exact Windows 11 ARM64 X Elite CRD requested CPU and returned an unsaturated waveform on the pinned fixture, matching local ONNX Runtime CPU with 8.09e-7 relative L2 and 121.84 dB SNR. Its 94-sample component profile had nearest-rank p50/p95 6.384/6.644 s and all 634 execution-detail rows on CPU; 5.917 s estimated time is the sample minimum. This is one component, without a listening-quality gate. Corrected DirectML inference and profile both failed at provider initialization with Windows error 80070057, producing no output or timing. A separate FP16 QNN DLC compile passed, but same-fixture NPU-requested inference was pending. Historical QNN talker profiling lacks a same-fixture numerical or full-pipeline gate. The older TFLite vocoder target was rejected on Windows ARM; no complete stream exists. | Audit the explicit QNN NPU inference, investigate the tested DirectML provider failure, and qualify talker/predictor plus complete local streaming, memory and sustained behavior; [vocoder evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md). |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_ONNX_CPU_VOCODER_PARITY_PROFILED; DIRECTML_RUNTIME_FAILED; TALKER_FP16_NUMERIC_FAIL; QNN_NPU_VOCODER_COMPILE_PASS; NOT_E2E_PROFILEABLE | The fresh ONNX vocoder inference on the exact Windows 11 ARM64 X Elite CRD requested CPU and returned an unsaturated waveform on the pinned fixture, matching local ONNX Runtime CPU with 8.09e-7 relative L2 and 121.84 dB SNR. Its 94-sample component profile had nearest-rank p50/p95 6.384/6.644 s and all 634 execution-detail rows on CPU; 5.917 s estimated time is the sample minimum. This is one component, without a listening-quality gate. Corrected DirectML inference and profile both failed at provider initialization with Windows error 80070057. A separate FP16 QNN DLC vocoder compile passed, but NPU-requested inference was pending. The 28-layer FP16 QNN talker step executed, yet changed CPU top-1 token 80 to 1720 with 8.57% logits relative L2 on a synthetic cache fixture; a separate ONNX CPU talker compile was submitted. The older TFLite vocoder target was rejected on Windows ARM; no complete stream exists. | Audit QNN vocoder and CPU talker controls, investigate DirectML failure, then qualify complete local streaming, memory and sustained behavior; [vocoder](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) and [talker](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) evidence. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | MiniCPM-o 4.5 | COMPONENT_SPEECH_HEAD_QNN_COMPILE_PASS; INFERENCE_PENDING; NOT_E2E_PROFILEABLE | The real 20-layer MiniCPM-o speech-head source compiled to FP16 QNN DLC for the exact X Elite CRD. Same-fixture NPU-requested inference was pending; no numerical output or placement profile exists. Thinker/encoders/vocoder and a device-local stateful Omni backend remain unqualified. | Audit the [X Elite speech-head inference](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md), then validate the complete multimodal pipeline, state, quality, memory and sustained behavior. |
|
||||
| PC Snapdragon X Elite CRD / AI Hub | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | Spark-X2.5 | NOT_E2E_PROFILEABLE | Fresh 2026-09-22 W8A16 full-attention component inference/profile passed; full1024 denotes one cache bucket. No full prefill/decode/KV/ring/sampling loop. | Implement and validate local 128-token generation, 512/1024 transitions, cancellation and resident memory. |
|
||||
| Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | Fresh FP16 predictor inference passed; its one output exactly matches the same compiled artifact/dataset reference. Historical decoder parity exists. No complete co-resident NPU/GPU stream. | Validate the device-local talker/predictor/vocoder pipeline, output quality, recovery and 30-minute thermal behavior. |
|
||||
| Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | Qwen3-TTS 0.6B CustomVoice | TALKER_FP16_NUMERIC_FAIL; NOT_E2E_PROFILEABLE | Fresh FP16 predictor inference passed; its one output exactly matches the same compiled artifact/dataset reference. The real 28-layer FP16 QNN talker step executed on the synthetic cache fixture, but changed CPU top-1 speech token 80 to 412 with 8.99% logits relative L2 and maximum new-K/V relative L2 11.71%. This tested talker artifact is numerically unqualified; no complete co-resident NPU/GPU stream exists. | Improve and revalidate the [talker artifact](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md), then qualify the device-local predictor/vocoder pipeline, output quality, recovery and 30-minute thermal behavior. |
|
||||
| Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | Fresh FP16 speech-head inference passed; all 41 outputs exactly match the same artifact/dataset reference. Historical FP16 numerical parity passed; tested W8A16 top-1 parity failed. Thinker/encoders/vocoder are not qualified together. | Supply a qualified complete mobile checkpoint/export and validate the full multimodal/stateful pipeline. |
|
||||
| Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | InternVLA-A1 | COMPONENT_NPU_ARTIFACT_COMPILE_PASS; INFERENCE_PENDING; NOT_E2E_PROFILEABLE | The real Place_Markpen Cosmos image encoder FP32 ONNX export matched its source on two six-frame fixtures (relative L2 8.74e-7/2.29e-6) after packaging external weights into a single file. Galaxy S25 FP16 QNN DLC compilation passed, but same-fixture NPU-requested inference was pending. There is no measured NPU placement, latent/action-policy parity, complete qualified target bundle or device-local controller; ADB has no attached device. | Audit the [S25 encoder inference](../../../e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md), then validate action quality, state, admission, cancellation and sustained behavior. |
|
||||
| Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | MiniCPM-o 4.5 | COMPONENT_FP16_SPEECH_HEAD_NUMERIC_MEASURED; NOT_E2E_PROFILEABLE | Fresh FP16 speech-head inference passed; all 41 outputs exactly matched the same artifact/dataset reference. Against the recovered original ONNX CPU source on the same fixture, logits relative L2 was 0.3605%, maximum new-K/V relative L2 1.311%, and top-1 speech token 1867 matched. Historical FP16 parity passed; tested W8A16 top-1 parity failed. No actual NPU placement profile, generated-audio quality gate, or qualified thinker/encoders/vocoder bundle exists. | Qualify device placement, a complete mobile checkpoint/export and the full multimodal/stateful pipeline; [speech-head evidence](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md). |
|
||||
| Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | InternVLA-A1 | COMPONENT_COSMOS_NUMERIC_MEASURED; SYNTHETIC_ACTION_SENSITIVITY_MEASURED; NOT_E2E_PROFILEABLE | The real Place_Markpen Cosmos image encoder FP32 ONNX export matched its source on two six-frame fixtures. Galaxy S25 FP16 QNN DLC inference returned a finite latent within 0.0524% relative L2 / 65.61 dB SNR of the pinned pattern source. Injecting this device latent into the same real policy on local CPU changed synthetic `[1,50,32]` actions by 0.742% relative L2; the CPU source repeated bitwise identically. No real-observation task tolerance or measured NPU placement exists (profile pending), and the policy was not resident on S25. ADB has no attached device. | Audit the [S25 encoder profile](../../../e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md), then validate real action quality, device-local state, admission, cancellation and sustained behavior. |
|
||||
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | Spark-X2.5 | COMPONENT_NPU_INFERENCE_PROFILED; NOT_E2E_PROFILEABLE | The real 1.7B checkpoint's fixed-shape full-attention decoder **one-layer** W8A16 QDQ artifact compiled as QNN DLC and ran on an exact hosted Galaxy S24 (Android 14). A same-dataset inference returned finite hidden/new-K/new-V outputs identical to the prior S25 NPU outputs; the profile attributed all 73 execution-detail rows to NPU. Across 100 component samples, nearest-rank p50/p95 was 1.109/1.205 ms; the 1.098 ms `estimated_inference_time` field is the minimum, not a median. Relative L2 versus original FP32 ONNX CPU was 1.351%/0.377%/0.650% for hidden/K/V; no token-quality tolerance is known. This is one cache bucket, not 28-layer prefill/decode, KV-ring, sampling, a resident local controller or complete-request timing. ADB has no attached device. | Qualify a complete device-resident artifact/controller with state and token-quality gates, 128-token generation across 512/1024 context transitions, admission/cancel and sustained memory/power; [S24 NPU raw evidence](../../../e2e_expansion_20260923/evidence/spark_s24_full_attention/README.md). |
|
||||
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | Historical FP16 decoder parity has SNR 39.13 dB. Tested calibrated INT8 decoder parity failed with negative SNR. Neither result is a full local TTS stream. | Validate the complete local pipeline and a quality-passing artifact set. |
|
||||
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | Qwen3-TTS 0.6B CustomVoice | TALKER_FP16_NUMERIC_FAIL; NOT_E2E_PROFILEABLE | The real 28-layer FP16 QNN talker step executed on the synthetic cache fixture, but changed CPU top-1 speech token 80 to 1720 with 9.33% logits relative L2 and maximum new-K/V relative L2 11.23%. This differs from an earlier FP16 decoder parity test (39.13 dB SNR) on a different component/fixture; the tested INT8 decoder parity failed with negative SNR. No full local TTS stream exists. | Resolve the [talker numerical difference](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md), then validate the complete local pipeline and a quality-passing artifact set. |
|
||||
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | Historical FP16 speech-head job re-fetched successfully; SNR 49.22 dB/top-1 match in the recorded parity test. W8A16 top-1 failed. Not complete MiniCPM-o. | Qualify thinker/encoders/vocoder and stateful local integration. |
|
||||
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | MiniCPM-o 4.5 | COMPONENT_FP16_SPEECH_HEAD_NUMERIC_MEASURED; NOT_E2E_PROFILEABLE | The retained FP16 speech-head job used a dataset with identical tensors to the recovered S25 fixture. Against original ONNX CPU, logits relative L2 was 0.3459%, maximum new-K/V relative L2 1.235%, and top-1 speech token 1867 matched. Earlier parity reported 49.22 dB SNR/top-1 match; tested W8A16 top-1 failed. Actual NPU placement, generated-audio quality and the rest of the model remain unqualified. | Qualify placement, thinker/encoders/vocoder and stateful local integration; [speech-head evidence](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md). |
|
||||
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Embedded SA8775P ADP / AI Hub | Spark-X2.5 | COMPONENT_NPU_INFERENCE_PROFILED; NOT_E2E_PROFILEABLE | A real Spark-X2.5-1.7B one-layer W8A16 QDQ attention artifact, compiled as QNN DLC, ran on the exact hosted SA8775P ADP (Android 14). Inference returned finite hidden/new-K/new-V tensors bitwise identical to the prior S24/S25 NPU outputs on the same fixture; the profile attributed all 73 execution-detail rows to NPU. Across 100 component samples, nearest-rank p50/p95 was 1.837/2.079 ms; the 1.749 ms estimated-time field is the minimum. Hidden/K/V relative L2 versus original FP32 ONNX CPU was 1.351%/0.377%/0.650%, without a next-token quality tolerance. There is no complete 28-layer prefill/decode/KV-ring/sampling loop, resident Omni controller or complete-request timing. | Qualify a complete embedded resident artifact/controller with state, token quality, memory admission/cancel and sustained power; [SA8775P NPU raw evidence](../../../e2e_expansion_20260923/evidence/spark_sa8775_full_attention/README.md). |
|
||||
| Embedded SA8775P ADP / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_CPU_REQUESTED_VOCODER_PARITY_PASS; COMPONENT_GPU_NUMERIC_FAIL; NOT_E2E_PROFILEABLE | On the exact SA8775P, the same TFLite binary's CPU-requested Code2Wav inference matched original ONNX CPU on the retained fixture within 4.13e-6 relative L2 / 107.67 dB SNR, with no saturation. Actual CPU placement/timing profile was pending. The GPU-requested output saturated 96.63% of samples, with 11.832 relative L2 / -21.46 dB SNR; its fresh 100-sample profile p50/p95 was 1.776/1.880 s with 403 CPU and 335 GPU execution-detail rows, timing a numerically failed waveform. Historical NPU talker profiling lacks a full pipeline. | Audit the pending CPU profile and FP32-preserving GPU control, establish listening quality, then validate co-resident talker/predictor/vocoder, streaming, memory and sustained behavior; [raw vocoder evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md). |
|
||||
| Embedded SA8775P ADP / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_CPU_REQUESTED_VOCODER_PARITY_PASS; COMPONENT_GPU_NUMERIC_FAIL; TALKER_FP16_NUMERIC_FAIL; NOT_E2E_PROFILEABLE | On the exact SA8775P, the same TFLite binary's CPU-requested Code2Wav inference matched original ONNX CPU within 4.13e-6 relative L2 / 107.67 dB SNR, without saturation; CPU placement/timing profile was pending. The GPU-requested output saturated 96.63% of samples with 11.832 relative L2; its fresh 100-sample mixed CPU/GPU profile p50/p95 was 1.776/1.880 s, timing failed audio. Even with FP32-preserving GPU options, 84.89% of samples saturated with 11.389 relative L2. The 28-layer FP16 QNN talker step executed but changed CPU top-1 token 80 to 1720 with 9.43% logits relative L2. No complete pipeline exists. | Isolate GPU delegate and talker numerical failures, audit pending CPU profile, then establish listening quality, co-resident streaming, memory and sustained behavior; [vocoder](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) and [talker](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) evidence. |
|
||||
| Embedded SA8775P ADP / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Embedded SA8775P ADP / AI Hub | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Embedded SA8775P ADP / AI Hub | MiniCPM-o 4.5 | COMPONENT_SPEECH_HEAD_QNN_COMPILE_PASS; INFERENCE_PENDING; NOT_E2E_PROFILEABLE | The real 20-layer MiniCPM-o speech-head source compiled to FP16 QNN DLC for the exact SA8775P ADP. Same-fixture NPU-requested inference was pending, so output parity and actual placement are unknown. No complete thinker/encoders/vocoder bundle or device-local stateful controller exists. | Audit the [SA8775P speech-head inference](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md), then qualify the complete multimodal pipeline and sustained behavior. |
|
||||
| Embedded SA8775P ADP / AI Hub | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | Spark-X2.5 | COMPONENT_CPU_PARITY_PASS; COMPONENT_NPU_NUMERIC_UNQUALIFIED; NOT_E2E_PROFILEABLE | On the exact hosted Dragonwing RB3 Gen 2 Vision Kit (QCS6490, QC Linux 1.6), the reused S24 W8A16 QNN DLC and a separately RB3-compiled W8A16 DLC each failed inference/profile at QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR. An FP32 ONNX one-layer CPU route passed with all 56 profile rows on CPU, source-relative hidden/K/V L2 of 8.73e-7/2.98e-7/3.37e-7 and 100-sample component p50/p95 19.304/21.113 ms. A calibrated W8A8 QNN DLC ran with all 61 rows on NPU and p50/p95 5.610/5.761 ms, but hidden/K/V relative L2 was 20.914%/1.702%/4.413% versus FP32 CPU, without a next-token quality gate. These are different precisions, not a qualified speedup; no complete 28-layer prefill/decode/KV-ring/sampling, local Omni controller or complete-request timing exists. | Improve representative W8A8 calibration or find another compatible quality-passing artifact; validate token parity, then full device-resident generation, admission/cancel and sustained power; [RB3 raw evidence](../../../e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md). |
|
||||
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_MIXED_VOCODER_PROFILED_NUMERIC_UNQUALIFIED; NOT_E2E_PROFILEABLE | Fresh same-fixture TFLite GPU-requested vocoder inference returned a finite unsaturated waveform, but relative L2 versus original ONNX CPU was 15.73% (16.06 dB SNR), without a listening-quality tolerance; it is not a qualified audio path. The fresh 100-sample component profile had nearest-rank p50/p95 3.099/3.133 s and 403 CPU plus 335 GPU execution-detail rows. Its 3.070 s estimated-time field is the sample minimum, not median. The historical NPU talker build was rejected for floating-point inputs; no full model pipeline exists. | Establish waveform quality and a compatible integer talker artifact, then validate complete local execution, memory and sustained behavior; [vocoder evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md). |
|
||||
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | MiniCPM-o 4.5 | COMPONENT_SPEECH_HEAD_QNN_COMPILE_PASS; INFERENCE_PENDING; NOT_E2E_PROFILEABLE | The real 20-layer MiniCPM-o speech-head source compiled to FP16 QNN DLC for the exact RB3 Gen 2 kit. Same-fixture NPU-requested inference was pending; RB3's older Spark/Qwen graph failures cannot substitute for this artifact's outcome. No numerical output, placement profile, complete model bundle or local stateful controller exists. | Audit the [RB3 speech-head inference](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md), then qualify the complete multimodal pipeline and sustained behavior. |
|
||||
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
|
||||
## Measured request timing
|
||||
|
||||
@@ -15,13 +15,15 @@ remain open; the 2026-09-22 audit remains the historical baseline.
|
||||
|
||||
The [Spark hosted Qualcomm component expansion](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_s24_full_attention/README.md) advances M1 only to one real fixed-shape decoder attention layer. The same W8A16 QNN DLC ran on S24, Snapdragon X Elite CRD and SA8775P ADP NPUs with identical outputs on one pinned fixture; S25 had prior separate component evidence. On [RB3 Gen 2](../../benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/spark_rb3_full_attention/README.md), the tested W8A16 graph failed QNN loading even after an exact-device compile. FP32 ONNX ran on CPU with near-source parity, while calibrated W8A8 ran on NPU but had 20.9% hidden-state relative L2 and remains numerically unqualified. These are component C results, not full M1: device-local 28-layer prefill/continuous decode, KV/ring, sampling, token quality, admission/cancellation, complete-request timing and sustained power/thermal gates remain open. AI Hub is the test facility, not a deployment dependency.
|
||||
|
||||
The [Qwen3-TTS Code2Wav Qualcomm follow-up](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) adds a retained same-fixture numerical check to M2. A fresh SA8775P GPU-requested output saturated 96.63% of samples and differs from original ONNX CPU by 11.832 relative L2, so that route fails the numerical gate despite profiling. The same binary's CPU-requested output on the exact SA device matched original ONNX CPU within 4.13e-6 relative L2; its placement profile remained pending. RB3 TFLite GPU-requested inference returned an unsaturated but 15.73%-relative-L2 divergent waveform; its fresh 100-sample mixed CPU/GPU component p50/p95 was 3.099/3.133 s. X Elite ONNX CPU inference matched the local ONNX Runtime output with 8.09e-7 relative L2; its 94-sample component profile was p50/p95 6.384/6.644 s with all execution-detail rows on CPU. Corrected DirectML inference and profile both failed during provider initialization; separate FP16 QNN DLC compilation passed with NPU-requested inference pending. SA FP32-preserving GPU control remained pending. M2 still requires a quality-passing talker/predictor/vocoder set, device-local co-residency and handoff, playable streaming and sustained memory/power validation.
|
||||
The [Qwen3-TTS Code2Wav Qualcomm follow-up](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) adds a retained same-fixture numerical check to M2. A fresh SA8775P GPU-requested output saturated 96.63% of samples and differs from original ONNX CPU by 11.832 relative L2, so that route fails the numerical gate despite profiling. The same binary's CPU-requested output on the exact SA device matched original ONNX CPU within 4.13e-6 relative L2; its placement profile remained pending. A FP32-preserving GPU-option control still saturated 84.89% of samples, so that option did not fix the tested route. RB3 TFLite GPU-requested inference returned an unsaturated but 15.73%-relative-L2 divergent waveform; its fresh 100-sample mixed CPU/GPU component p50/p95 was 3.099/3.133 s. X Elite ONNX CPU inference matched the local ONNX Runtime output with 8.09e-7 relative L2; its 94-sample component profile was p50/p95 6.384/6.644 s with all execution-detail rows on CPU. Corrected DirectML inference and profile both failed during provider initialization; separate FP16 QNN DLC compilation passed with NPU-requested inference pending. M2 still requires a quality-passing talker/predictor/vocoder set, device-local co-residency and handoff, playable streaming and sustained memory/power validation.
|
||||
|
||||
The identical SA/RB3 TFLite vocoder binary also matched original ONNX CPU within 3.45e-6 relative L2 on a separate local HX370 x86-64 LiteRT 2.2.0 CPU control. Exact-SA CPU-requested parity now narrows the failure further to the GPU-requested route. Its per-node placement and the FP32-preserving GPU control are the next isolation gates; neither identifies the failing delegate operation yet.
|
||||
The identical SA/RB3 TFLite vocoder binary also matched original ONNX CPU within 3.45e-6 relative L2 on a separate local HX370 x86-64 LiteRT 2.2.0 CPU control. Exact-SA CPU-requested parity narrows the failure to the GPU-requested route. The FP32-preserving GPU option did not fix it, so per-node placement and delegate-operation isolation remain open.
|
||||
|
||||
A separate [28-layer talker decode-step attempt](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) retains a synthetic fixed-cache fixture and CPU reference. Historical FP16 QNN context-binary compiles passed for S24, S25, SA8775P and X Elite; their same-fixture NPU inference was pending at this snapshot. RB3 rejected that floating-point-input artifact. None advances M2 to numerical talker or full-stream qualification yet.
|
||||
A separate [28-layer talker decode-step attempt](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) retains a synthetic fixed-cache fixture and CPU reference. Historical FP16 QNN context-binary inference executed on S24, S25, SA8775P and X Elite, but all four changed the CPU top-1 speech token with 8.57–9.43% logits relative L2. RB3 rejected that floating-point-input artifact. An explicit X Elite ONNX CPU fallback compile was pending. None advances M2 to numerical talker or full-stream qualification yet.
|
||||
|
||||
For M4, a [Galaxy S25 Cosmos encoder attempt](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md) repackages the real Place_Markpen FP32 ONNX export after retaining two CPU parity fixtures. FLOAT16 QNN DLC compilation passed and same-fixture NPU-requested inference was pending. This is only an artifact feasibility gate, not an NPU execution or full-policy result.
|
||||
For M4, a [MiniCPM-o 4.5 speech-head study](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md) recovered the historical 20-layer ONNX source and matching S24/S25 fixtures. Both FP16 context-binary outputs preserved the ONNX CPU top-1 speech token on one decode step, with logits relative L2 0.346%/0.360%. New FP16 QNN DLC compiles passed for SA8775P, X Elite and RB3, while same-fixture NPU-requested inference remained pending. This is component C numerical evidence only, not thinker/encoder/vocoder or complete multimodal support.
|
||||
|
||||
For M4, a [Galaxy S25 Cosmos encoder attempt](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md) repackages the real Place_Markpen FP32 ONNX export after retaining two CPU parity fixtures. FLOAT16 QNN DLC inference returned a latent within 0.0524% relative L2 of the source on one synthetic pattern. Injecting that output into the real policy on local CPU changed synthetic actions by 0.742% relative L2, without a task tolerance. NPU placement profile was pending, and the policy did not run on S25. This is component/action-sensitivity evidence, not device-local M4 qualification.
|
||||
|
||||
The [WSL CPU expansion](../../benchmarks/edge_harness/results/e2e_expansion_20260923/README.md)
|
||||
adds constrained three-stage MiniCPM-o text-to-speech requests and a CPU-only
|
||||
|
||||
Reference in new issue
Block a user