Extend Cosmos and TTS component checks to S24 and RB3

This commit is contained in:
tzhouam committed 2026-09-24 09:51:08 +08:00
1 parent 2ff0a783d4
commit 05210fb2db
25 files changed
+468 -11

No files matched your search

@@ -0,0 +1,87 @@
#!/usr/bin/env python3
"""Join a complete sustained TTS request run with device-wide GPU telemetry."""
from __future__ import annotations
import argparse
import json
from collections import Counter
from pathlib import Path
from analyze_tts_sustained import WINDOWS_MIN, nearest_rank, summarize
METRICS = (
"gpu_temperature_c",
"gpu_power_mw",
"gpu_clock_sm_mhz",
"gpu_clock_memory_mhz",
"gpu_utilization_pct",
"gpu_memory_used_bytes",
"host_available_bytes",
"host_cpu_utilization_pct",
)
def analyze(report_path: Path, request_path: Path, telemetry_path: Path) -> dict:
report = json.loads(report_path.read_text(encoding="utf-8"))
if report.get("status") != "completed" or report.get("sustained_wall_s", 0) < 1800:
raise ValueError("A terminal 30-minute sustained report is required")
if report.get("gpu_telemetry", {}).get("error"):
raise ValueError("GPU telemetry reported an error")
if report.get("gpu_telemetry", {}).get("samples", 0) < 1500:
raise ValueError("Too few device telemetry samples for a 30-minute run")
start = report["sustained_start_unix"]
base = summarize(request_path)
telemetry = [json.loads(line) for line in telemetry_path.open(encoding="utf-8")]
if len(telemetry) != report["gpu_telemetry"]["samples"]:
raise ValueError("Telemetry sample count differs from the final report")
for (lo, hi), window in zip(WINDOWS_MIN, base["windows"], strict=True):
samples = [row for row in telemetry if lo * 60 <= row["unix"] - start < hi * 60]
window["device_telemetry_samples"] = len(samples)
window["device_telemetry"] = {}
for name in METRICS:
values = [row[name] for row in samples if row.get(name) is not None]
window["device_telemetry"][name] = {
"p50": nearest_rank(values, 0.5),
"p95": nearest_rank(values, 0.95),
"min": min(values) if values else None,
"max": max(values) if values else None,
"available_samples": len(values),
}
window["gpu_throttle_reason_counts"] = dict(sorted(Counter(
str(row["gpu_throttle_reasons"]) for row in samples
).items()))
return {
"scope": "complete-model Qwen3-TTS stream; simulated playback; telemetry covers whole GPU/host, not just this process",
"report": str(report_path),
"requests": str(request_path),
"telemetry": str(telemetry_path),
"loaded_omni_path": report.get("loaded_omni_path"),
"cwd": report.get("cwd"),
"start_unix": start,
"sustained_wall_s": report["sustained_wall_s"],
"windows": base["windows"],
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--run-dir", required=True, type=Path)
parser.add_argument("--out", type=Path)
args = parser.parse_args()
result = analyze(
args.run_dir / "report.json",
args.run_dir / "requests.jsonl",
args.run_dir / "gpu_telemetry.jsonl",
)
output = json.dumps(result, indent=2) + "\n"
if args.out:
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(output, encoding="utf-8")
else:
print(output, end="")
if __name__ == "__main__":
main()
@@ -33,6 +33,8 @@ def main() -> None:
parser.add_argument(f"--{name}", type=Path, required=True)
parser.add_argument("--device-name", default="Samsung Galaxy S25")
parser.add_argument("--profile-report", type=Path)
parser.add_argument("--expected-compile-options", default=COMPILE_OPTIONS)
parser.add_argument("--expected-run-options", default="--compute_unit npu")
args = parser.parse_args()
compiled = json.loads(args.compile_report.read_text(encoding="utf-8"))
@@ -40,7 +42,8 @@ def main() -> None:
inferred = json.loads(args.inference_report.read_text(encoding="utf-8"))
if compiled.get("status") != "SUCCESS" or inferred.get("status") != "SUCCESS":
raise ValueError("compile and inference must be terminal successes")
if compiled.get("source_model_id") != "mn4okj3rq" or compiled.get("options") != COMPILE_OPTIONS:
if (compiled.get("source_model_id") != "mn4okj3rq"
or compiled.get("options") != args.expected_compile_options):
raise ValueError("compile used a different source or options")
if inferred.get("model_id") != compiled.get("target_model_id"):
raise ValueError("inference used a different target model")
@@ -48,8 +51,8 @@ def main() -> None:
raise ValueError("compile/inference device metadata differs")
if inferred.get("device", {}).get("name") != args.device_name:
raise ValueError("inference did not use the requested exact device")
if inferred.get("options") != "--compute_unit npu":
raise ValueError("inference did not request NPU")
if inferred.get("options") != args.expected_run_options:
raise ValueError("inference did not request the expected compute unit")
if (inferred.get("job_id") != submitted.get("job_id")
or inferred.get("input_dataset_id") != submitted.get("input_dataset_id")):
raise ValueError("inference used a different job or input dataset")
@@ -121,7 +124,7 @@ def main() -> None:
if (profiled.get("status") != "SUCCESS"
or profiled.get("model_id") != compiled["target_model_id"]
or profiled.get("device") != inferred["device"]
or profiled.get("options") != "--compute_unit npu"):
or profiled.get("options") != args.expected_run_options):
raise ValueError("profile did not execute the same target on the exact device")
profile = profiled["profile"]
summary = profile["execution_summary"]
@@ -47,9 +47,12 @@ def collect(evidence_dir: Path, kind: str, client: hub.Client) -> str:
"options": job.options,
"hub_version": str(job.hub_version),
"device": device_record(job.device),
"model_id": job.model.model_id,
}
if kind == "inference":
if kind == "compile":
report["source_model_id"] = job.model.model_id
report["target_model_id"] = job.get_target_model().model_id if status.code == "SUCCESS" else None
elif kind == "inference":
report["model_id"] = job.model.model_id
report["input_dataset_id"] = job.inputs.dataset_id
if status.code == "SUCCESS":
output_path = evidence_dir / "device_output.npz"
@@ -58,6 +61,7 @@ def collect(evidence_dir: Path, kind: str, client: hub.Client) -> str:
np.savez_compressed(output_path, **arrays)
report["outputs"] = output_path.name
elif kind == "profile":
report["model_id"] = job.model.model_id
report["shapes"] = job.shapes
if status.code == "SUCCESS":
report["profile"] = job.download_profile()
@@ -70,7 +74,7 @@ def collect(evidence_dir: Path, kind: str, client: hub.Client) -> str:
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("evidence_dirs", nargs="+", type=Path)
parser.add_argument("--kinds", nargs="+", choices=("inference", "profile"), default=("inference", "profile"))
parser.add_argument("--kinds", nargs="+", choices=("compile", "inference", "profile"), default=("inference", "profile"))
args = parser.parse_args()
client = hub.Client()
for directory in args.evidence_dirs:
@@ -11,3 +11,5 @@ The [MiniCPM-o speech-head study](evidence/minicpmo_tts_qualcomm/README.md) reco
The [native Windows Qwen3-TTS sustained analysis](evidence/qwen_tts_sustained_drift/README.md) finds a sharp playback throughput collapse around minute 26–27 in the historical 30-minute RTX 5090 Laptop stream. The last 17 serial requests all had simulated underruns with median RTF 1.331; the separate WSL run did not show that behavior. A revised local profiler now records timestamped device-wide NVML clocks, power, temperature, throttling and utilization to investigate the cause in a repeat run. The historical evidence cannot identify the cause or qualify native Windows sustained playback.
The [InternVLA Cosmos S25 attempt](evidence/internvla_cosmos_qualcomm/README.md) repackages the real policy's image encoder from its parity-tested ONNX export. FLOAT16 QNN DLC inference returned a latent within 0.0524% relative L2 of the source on one pattern. Injecting it into the real policy on local CPU changed synthetic actions by 0.742% relative L2, without a task tolerance. S25 six-frame component placement profile returned 100 samples at p50/p95 49.665/51.382 ms, with all 121 execution-detail rows on NPU. The same encoder compiled to FP16 QNN DLC for X Elite CRD; same-fixture NPU-requested inference returned a finite latent at 0.0608% relative L2 / 64.32 dB SNR versus the source, with its 100-sample six-frame profile at p50/p95 82.939/83.837 ms and all 121 execution-detail rows on NPU. An SA8775P FP16 QNN DLC inference returned a finite same-fixture latent at 0.0613% relative L2 / 64.25 dB SNR versus source; its 100-sample six-frame profile measured p50/p95 116.021/116.327 ms with all 121 execution-detail rows on NPU. Device-local action-policy support remains unqualified. The checkpoint-named Place_Markpen dataset returned HTTP 404, and the official vLLM-Omni example archive in InternData-A1 returned HTTP 403 gated-access denial under the current authenticated account; [real-observation access evidence](evidence/internvla_real_observation_access/README.md) records the blocker without treating synthetic actions as task validation.
New exact-device [Galaxy S24 QNN](evidence/internvla_cosmos_qualcomm/s24/audit_report.json) and [RB3 ONNX CPU](evidence/internvla_cosmos_qualcomm/rb3_cpu/audit_report.json) Cosmos encoder inferences returned finite same-fixture latents at 0.0613% and 2.20e-6 relative L2, respectively. Placement profiles remain pending. Neither result runs the action policy locally or establishes a real-observation task tolerance.
@@ -8,6 +8,8 @@ The same source model also [compiled to FP16 QNN DLC on Snapdragon X Elite CRD](
The same source also [compiled to FP16 QNN DLC on SA8775P ADP](sa8775p/compile_report.json), producing target mn750wz8m for Android. [Same-fixture NPU-requested inference](sa8775p/inference_report.json) using dataset d26qo8p57 returned a finite FP32 [6,16,32,32] latent. Its [audit](sa8775p/audit_report.json) measured **0.0613% relative L2 / 64.25 dB SNR** versus the pinned source pattern, with cosine 0.999999812. The [placement profile](sa8775p/profile_report.json) returned 100 six-frame component samples at nearest-rank p50/p95 **116.021/116.327 ms**, with all **121 execution-detail rows on NPU**. The 114.779 ms estimated-time field is the sample minimum. Reported inference peak memory was 116,924,416 bytes and first-load peak 659,845,120 bytes. No complete local policy follows.
The same source has exact-device [Galaxy S24 FP16 QNN DLC](s24/compile_report.json) and [RB3 Gen 2 ONNX CPU](rb3_cpu/compile_report.json) compiles. Same-fixture [S24 NPU-requested inference](s24/inference_report.json) returned a finite latent at **0.0613% relative L2 / 64.25 dB SNR** versus the pinned source pattern ([audit](s24/audit_report.json)). [RB3 CPU-requested inference](rb3_cpu/inference_report.json) returned a finite latent at **2.20e-6 relative L2 / 113.17 dB SNR** versus source ([audit](rb3_cpu/audit_report.json)). Their [S24 NPU](s24/profile_submission.json) and [RB3 CPU](rb3_cpu/profile_submission.json) placement profiles are pending. These are encoder component checks; no complete policy was resident on either device.
The [generic collector](../../../../experiments/collect_aihub_component.py) preserves terminal jobs, and the [Cosmos auditor](../../../../experiments/audit_internvla_cosmos_qualcomm.py) checks source/fixture hashes, target/device/dataset identity, latent shape and finite numerical difference against the pinned source reference. A separate [synthetic action sensitivity check](s25/action_sensitivity.json) injected either the pinned source or S25 latent into the same real Place_Markpen policy on **local CPU**, with zero state/noise. The [reproducer](../../../../experiments/probe_internvla_hosted_latent_actions.py) returned finite `[1,50,32]` actions; hosted-vs-source action relative L2 was **0.742%**, maximum absolute difference **0.0115**, cosine **0.999973**. Repeating the source policy gave bitwise identical actions. This is a cross-device numerical handoff experiment, not a device-local policy or robot-task pass; no real-observation action tolerance is known.
Next: test representative real observations and action tolerances, then build and validate a device-local stateful policy. Full multimodal observation processing, admission, cancellation and sustained profiling remain open.
@@ -0,0 +1,54 @@
{
"scope": "one six-frame Cosmos image encoder on synthetic pattern; no complete InternVLA policy",
"status": "component_inference_numeric_measured",
"device": {
"name": "Dragonwing RB3 Gen 2 Vision Kit",
"os": "1.6",
"attributes": [
"os:qc_linux",
"framework:tflite",
"framework:onnx",
"vendor:qualcomm",
"abi:aarch64-oe-linux-gcc11",
"format:iot",
"chipset:qualcomm-qcs6490",
"chipset:qcs6490",
"chipset:qualcomm-dragonwing-qcs6490",
"hexagon:v68",
"soc-model:93",
"framework:qnn"
]
},
"source_model_id": "mn4okj3rq",
"target_model_id": "mno4105gm",
"compile_job_id": "j56878x6g",
"inference_job_id": "jpyoxo0l5",
"input_dataset_id": "d26qo8p57",
"requested_run_options": "--compute_unit cpu",
"files": {
"source_onnx": {
"bytes": 125112088,
"sha256": "656a5263f584851a1457ef6dcf60b995f7c67b056b3c90a4f33c814f7caf9ec9"
},
"fixture": {
"bytes": 2081321,
"sha256": "dcbaf1391ca914d4bbccc1cb384daf05666e771d9cdae90c4f069ec8fcf05368"
},
"device_output": {
"bytes": 357883,
"sha256": "a316ab251ff5257f9af60e80e44ea81ff6da07b83d84b618ae8837a160c83cdc"
}
},
"output_dtype": "float32",
"source_vs_device": {
"relative_l2": 2.196231135016446e-06,
"snr_db": 113.16643911886588,
"max_abs_error": 2.6702880859375e-05,
"cosine": 0.9999999999978003
},
"limits": [
"The fixture is synthetic; no real-observation or action-quality tolerance is established.",
"The action policy, state lifecycle, admission and complete request were not run on this device.",
"Requested NPU is not actual per-node placement without a device profile."
]
}
@@ -0,0 +1,31 @@
{
"timestamp_utc": "2026-09-24T01:41:37.263839+00:00",
"qai_hub_version": "0.55.0",
"facility": "Qualcomm AI Hub Workbench",
"job_id": "j56878x6g",
"url": "https://workbench.aihub.qualcomm.com/jobs/j56878x6g/",
"status": "SUCCESS",
"status_message": "",
"options": "--target_runtime onnx",
"hub_version": "aihub-2026.09.11.0",
"device": {
"name": "Dragonwing RB3 Gen 2 Vision Kit",
"os": "1.6",
"attributes": [
"os:qc_linux",
"framework:tflite",
"framework:onnx",
"vendor:qualcomm",
"abi:aarch64-oe-linux-gcc11",
"format:iot",
"chipset:qualcomm-qcs6490",
"chipset:qcs6490",
"chipset:qualcomm-dragonwing-qcs6490",
"hexagon:v68",
"soc-model:93",
"framework:qnn"
]
},
"source_model_id": "mn4okj3rq",
"target_model_id": "mno4105gm"
}
@@ -0,0 +1,17 @@
{
"timestamp_utc": "2026-09-24T01:39:57.129903+00:00",
"facility": "Qualcomm AI Hub Workbench",
"qai_hub_version": "0.55.0",
"source_model_id": "mn4okj3rq",
"source_artifact": {
"name": "cosmos_encoder_embedded.onnx",
"bytes": 125112088,
"sha256": "656a5263f584851a1457ef6dcf60b995f7c67b056b3c90a4f33c814f7caf9ec9"
},
"device": "Dragonwing RB3 Gen 2 Vision Kit",
"options": "--target_runtime onnx",
"job_id": "j56878x6g",
"url": "https://workbench.aihub.qualcomm.com/jobs/j56878x6g/",
"kind": "compile",
"status": "SUBMITTED"
}
@@ -0,0 +1,32 @@
{
"timestamp_utc": "2026-09-24T01:48:38.816335+00:00",
"qai_hub_version": "0.55.0",
"facility": "Qualcomm AI Hub Workbench",
"job_id": "jpyoxo0l5",
"url": "https://workbench.aihub.qualcomm.com/jobs/jpyoxo0l5/",
"status": "SUCCESS",
"status_message": "",
"options": "--compute_unit cpu",
"hub_version": "aihub-2026.09.11.0",
"device": {
"name": "Dragonwing RB3 Gen 2 Vision Kit",
"os": "1.6",
"attributes": [
"os:qc_linux",
"framework:tflite",
"framework:onnx",
"vendor:qualcomm",
"abi:aarch64-oe-linux-gcc11",
"format:iot",
"chipset:qualcomm-qcs6490",
"chipset:qcs6490",
"chipset:qualcomm-dragonwing-qcs6490",
"hexagon:v68",
"soc-model:93",
"framework:qnn"
]
},
"model_id": "mno4105gm",
"input_dataset_id": "d26qo8p57",
"outputs": "device_output.npz"
}
@@ -0,0 +1,14 @@
{
"timestamp_utc": "2026-09-24T01:42:39.558439+00:00",
"facility": "Qualcomm AI Hub Workbench",
"qai_hub_version": "0.55.0",
"source_compile_job_id": "j56878x6g",
"target_model_id": "mno4105gm",
"input_dataset_id": "d26qo8p57",
"device": "Dragonwing RB3 Gen 2 Vision Kit",
"options": "--compute_unit cpu",
"kind": "inference",
"status": "SUBMITTED",
"job_id": "jpyoxo0l5",
"url": "https://workbench.aihub.qualcomm.com/jobs/jpyoxo0l5/"
}
@@ -0,0 +1,13 @@
{
"timestamp_utc": "2026-09-24T01:48:56.671423+00:00",
"facility": "Qualcomm AI Hub Workbench",
"qai_hub_version": "0.55.0",
"source_compile_job_id": "j56878x6g",
"target_model_id": "mno4105gm",
"device": "Dragonwing RB3 Gen 2 Vision Kit",
"options": "--compute_unit cpu",
"kind": "profile",
"status": "SUBMITTED",
"job_id": "jpvld2475",
"url": "https://workbench.aihub.qualcomm.com/jobs/jpvld2475/"
}
@@ -0,0 +1,55 @@
{
"scope": "one six-frame Cosmos image encoder on synthetic pattern; no complete InternVLA policy",
"status": "component_inference_numeric_measured",
"device": {
"name": "Samsung Galaxy S24",
"os": "14",
"attributes": [
"os:android",
"framework:tflite",
"framework:onnx",
"abi:aarch64-android",
"vendor:samsung",
"format:phone",
"chipset:qualcomm-snapdragon-8gen3",
"chipset:sm8650",
"hexagon:v75",
"soc-model:57",
"htp-supports-fp16:true",
"htp-supports-weight-sharing:true",
"framework:qnn"
]
},
"source_model_id": "mn4okj3rq",
"target_model_id": "mq33p123q",
"compile_job_id": "jglyxydj5",
"inference_job_id": "jp2rwrnqg",
"input_dataset_id": "d26qo8p57",
"requested_run_options": "--compute_unit npu",
"files": {
"source_onnx": {
"bytes": 125112088,
"sha256": "656a5263f584851a1457ef6dcf60b995f7c67b056b3c90a4f33c814f7caf9ec9"
},
"fixture": {
"bytes": 2081321,
"sha256": "dcbaf1391ca914d4bbccc1cb384daf05666e771d9cdae90c4f069ec8fcf05368"
},
"device_output": {
"bytes": 185671,
"sha256": "f15119b8dbeed9a5fbccd2d918dc52277606ee8bbdbf81cfdd16c77c1d24060a"
}
},
"output_dtype": "float32",
"source_vs_device": {
"relative_l2": 0.0006133847737886196,
"snr_db": 64.24534017651494,
"max_abs_error": 0.009660720825195312,
"cosine": 0.9999998119817205
},
"limits": [
"The fixture is synthetic; no real-observation or action-quality tolerance is established.",
"The action policy, state lifecycle, admission and complete request were not run on this device.",
"Requested NPU is not actual per-node placement without a device profile."
]
}
@@ -0,0 +1,32 @@
{
"timestamp_utc": "2026-09-24T01:41:36.020967+00:00",
"qai_hub_version": "0.55.0",
"facility": "Qualcomm AI Hub Workbench",
"job_id": "jglyxydj5",
"url": "https://workbench.aihub.qualcomm.com/jobs/jglyxydj5/",
"status": "SUCCESS",
"status_message": "",
"options": "--target_runtime qnn_dlc --qnn_options default_graph_htp_precision=FLOAT16",
"hub_version": "aihub-2026.09.11.0",
"device": {
"name": "Samsung Galaxy S24",
"os": "14",
"attributes": [
"os:android",
"framework:tflite",
"framework:onnx",
"abi:aarch64-android",
"vendor:samsung",
"format:phone",
"chipset:qualcomm-snapdragon-8gen3",
"chipset:sm8650",
"hexagon:v75",
"soc-model:57",
"htp-supports-fp16:true",
"htp-supports-weight-sharing:true",
"framework:qnn"
]
},
"source_model_id": "mn4okj3rq",
"target_model_id": "mq33p123q"
}
@@ -0,0 +1,17 @@
{
"timestamp_utc": "2026-09-24T01:39:55.375798+00:00",
"facility": "Qualcomm AI Hub Workbench",
"qai_hub_version": "0.55.0",
"source_model_id": "mn4okj3rq",
"source_artifact": {
"name": "cosmos_encoder_embedded.onnx",
"bytes": 125112088,
"sha256": "656a5263f584851a1457ef6dcf60b995f7c67b056b3c90a4f33c814f7caf9ec9"
},
"device": "Samsung Galaxy S24",
"options": "--target_runtime qnn_dlc --qnn_options default_graph_htp_precision=FLOAT16",
"job_id": "jglyxydj5",
"url": "https://workbench.aihub.qualcomm.com/jobs/jglyxydj5/",
"kind": "compile",
"status": "SUBMITTED"
}
@@ -0,0 +1,33 @@
{
"timestamp_utc": "2026-09-24T01:47:43.286574+00:00",
"qai_hub_version": "0.55.0",
"facility": "Qualcomm AI Hub Workbench",
"job_id": "jp2rwrnqg",
"url": "https://workbench.aihub.qualcomm.com/jobs/jp2rwrnqg/",
"status": "SUCCESS",
"status_message": "",
"options": "--compute_unit npu",
"hub_version": "aihub-2026.09.11.0",
"device": {
"name": "Samsung Galaxy S24",
"os": "14",
"attributes": [
"os:android",
"framework:tflite",
"framework:onnx",
"abi:aarch64-android",
"vendor:samsung",
"format:phone",
"chipset:qualcomm-snapdragon-8gen3",
"chipset:sm8650",
"hexagon:v75",
"soc-model:57",
"htp-supports-fp16:true",
"htp-supports-weight-sharing:true",
"framework:qnn"
]
},
"model_id": "mq33p123q",
"input_dataset_id": "d26qo8p57",
"outputs": "device_output.npz"
}
@@ -0,0 +1,14 @@
{
"timestamp_utc": "2026-09-24T01:42:37.521493+00:00",
"facility": "Qualcomm AI Hub Workbench",
"qai_hub_version": "0.55.0",
"source_compile_job_id": "jglyxydj5",
"target_model_id": "mq33p123q",
"input_dataset_id": "d26qo8p57",
"device": "Samsung Galaxy S24",
"options": "--compute_unit npu",
"kind": "inference",
"status": "SUBMITTED",
"job_id": "jp2rwrnqg",
"url": "https://workbench.aihub.qualcomm.com/jobs/jp2rwrnqg/"
}
@@ -0,0 +1,13 @@
{
"timestamp_utc": "2026-09-24T01:48:11.142086+00:00",
"facility": "Qualcomm AI Hub Workbench",
"qai_hub_version": "0.55.0",
"source_compile_job_id": "jglyxydj5",
"target_model_id": "mq33p123q",
"device": "Samsung Galaxy S24",
"options": "--compute_unit npu",
"kind": "profile",
"status": "SUBMITTED",
"job_id": "jpe7z6475",
"url": "https://workbench.aihub.qualcomm.com/jobs/jpe7z6475/"
}
@@ -8,6 +8,8 @@ Historical FP16 QNN context-binary [S24](s24/compile_report.json), [S25](s25/com
Explicit [X Elite ONNX CPU](xelite/cpu_fallback/compile_report.json) and [SA8775P TFLite CPU](sa8775p/cpu_fallback/compile_report.json) fallback compiles succeeded from the same source. The same-fixture [X Elite CPU-requested inference](xelite/cpu_fallback/inference_report.json) succeeded. Its [audit](xelite/cpu_fallback/audit_report.json) found logits relative L2 2.07e-5, maximum new-K/V relative L2 2.18e-5, and preserved ONNX CPU top-1 speech token 80. The [CPU-requested placement profile](xelite/cpu_fallback/profile_report.json) returned 100 decode-step component samples at nearest-rank p50/p95 **191.414/309.428 ms**, with all **1,744 execution-detail rows on CPU**. The 142.921 ms estimated-time field is the sample minimum; reported inference peak memory was 1,928,019,968 bytes. The [SA8775P TFLite CPU-requested inference](sa8775p/cpu_fallback/inference_report.json) also succeeded. Its [audit](sa8775p/cpu_fallback/audit_report.json) found logits relative L2 3.33e-5, maximum new-K/V relative L2 3.41e-5, and preserved CPU top-1 speech token 80. The [CPU-requested placement profile](sa8775p/cpu_fallback/profile_report.json) returned 100 decode-step component samples at nearest-rank p50/p95 **52.635/76.925 ms**, with all **1,800 execution-detail rows on CPU**. The 45.688 ms estimated-time field is the sample minimum; reported inference peak memory was 1,913,794,560 bytes. Both explicit CPU routes are numerically close on one decode step, not complete talker loops or TTS deployments.
An exact RB3 Gen 2 [TFLite CPU fallback compile](rb3/cpu_fallback/compile_submission.json) has been submitted from the same ONNX talker step. Its outcome is pending; the earlier RB3 FP16 QNN rejection remains the result for that specific NPU artifact.
The [collector](../../../../experiments/collect_aihub_component.py) retrieves terminal inference outputs without resubmission. The [tensor auditor](../../../../experiments/audit_qwen_tts_talker_qualcomm.py) enforces the source, fixture, compile target, exact device, output shape and finiteness contracts, and reports logits top-1 plus relative error for all 56 new K/V tensors. It does not convert those metrics to a quality pass without an explicit token/audio tolerance. Its 57-output contract was exercised against the pinned CPU reference as a local self-check; that self-check is not device evidence. Requested NPU placement still needs a profile.
Next: investigate FP16 numerical drift using realistic generated-cache fixtures or a higher-precision/mixed-precision artifact, and inspect placement only for a quality-passing decode step. Full device-local talker/predictor/vocoder streaming and power/memory qualification remain separate gates.
@@ -0,0 +1,12 @@
{
"timestamp_utc": "2026-09-24T01:44:12.754578+00:00",
"facility": "Qualcomm AI Hub Workbench",
"qai_hub_version": "0.55.0",
"source_model_id": "mq3xre4rq",
"device": "Dragonwing RB3 Gen 2 Vision Kit",
"options": "--target_runtime tflite",
"kind": "compile",
"status": "SUBMITTED",
"job_id": "j5687o2yg",
"url": "https://workbench.aihub.qualcomm.com/jobs/j5687o2yg/"
}
@@ -14,6 +14,8 @@ On the **exact hosted SA8775P**, a [CPU-requested inference](sa8775p/cpu_control
The re-fetched *historical* [SA8775P profile](sa8775p/historical_profile_report.json) has 100 component samples, nearest-rank p50/p95 **1.751/1.874 s**, and 403 CPU plus 335 GPU execution-detail rows. The *historical* [RB3 profile](rb3/historical_profile_report.json) has 100 samples, p50/p95 **3.143/3.189 s**, also with 403 CPU plus 335 GPU rows. Their previously quoted 1.701/3.107 s estimated-time fields are sample minima, not medians. Both profiles exclude talker/predictor, stage handoff, streaming and full-request time.
An exact RB3 Gen 2 [CPU-requested control inference](rb3/cpu_control/inference_submission.json) has been submitted using the same TFLite artifact and pinned waveform fixture as the GPU-requested run. Its numerical result is pending; the GPU-requested waveform remains unqualified.
The X Elite [ONNX CPU inference](xelite/cpu/inference_report.json) succeeded on the pinned fixture. Its [waveform comparison](xelite/cpu/inference_comparison.json) to the original local ONNX Runtime CPU output was **8.09e-7 relative L2 / 121.84 dB SNR**, with no saturation. The fresh [CPU profile](xelite/cpu/profile_report.json) returned **94** component samples, nearest-rank p50/p95 **6.384/6.644 s**, with all **634** execution-detail rows on CPU; the service's 5.917 s estimated-time field is the sample minimum. The [full audit](xelite/cpu/audit_report.json) pins source, fixture, target, waveform, placement and percentiles. This is near-source component parity on one fixture, without a listening-quality or complete-stream gate. The first X Elite [GPU inference](xelite/gpu/inference_report.json) and [profile](xelite/gpu/profile_report.json) jobs failed because Workbench rejects specifying both compute unit and ONNX execution provider. The corrected [DirectML inference](xelite/directml/inference_report.json) and [profile](xelite/directml/profile_report.json) both failed during DML provider initialization with Windows error `80070057` (`The parameter is incorrect`), without an output or timing samples. A separate [FP16 QNN DLC compile](xelite/qnn_npu/compile_report.json) succeeded. Its [NPU-requested same-fixture inference](xelite/qnn_npu/inference_report.json) returned a finite, unsaturated waveform at **1.138% relative L2 / 38.88 dB SNR** versus original ONNX CPU ([comparison](xelite/qnn_npu/inference_comparison.json)). This is better waveform agreement on this fixture than the historical S25 GPU output, but lacks a listening-quality tolerance and verified NPU node placement. Its [placement profile](xelite/qnn_npu/profile_report.json) failed after exceeding Workbench maximum execution time while preparing the graph; no timing or per-node placement samples were returned. The [compressed runtime log](xelite/qnn_npu/logs/jpyoxmvr5_runtime.log.gz) retains the QNN HTP preparation/serialization trace. The [X Elite ONNX CPU compile](xelite/compile_report.json) passed after the older TFLite target was rejected on Windows ARM. These failures constrain the tested DirectML route, not all GPU or NPU artifacts.
A new SA8775P [FP16 QNN DLC compile](sa8775p/qnn_npu/compile_report.json) passed from the same source ONNX, producing target `mqvovl9lm`; [same-fixture NPU-requested inference](sa8775p/qnn_npu/inference_report.json) failed with Workbench message "Failed to fully run the model, failed after compiling." It returned no waveform or placement samples. This separate accelerator artifact therefore does not rescue the numerically failed TFLite GPU route; the measured CPU component route remains available. The failure applies to this FP16 QNN DLC artifact, not every potential NPU export. The [collector](../../../../experiments/collect_aihub_component.py) retrieves terminal inference/profile reports without resubmitting jobs, and the [audit script](../../../../experiments/audit_qwen_tts_vocoder_qualcomm.py) checks source/fixture hashes, device and model IDs, actual placement, waveform error, saturation and sample percentiles.
@@ -0,0 +1,14 @@
{
"timestamp_utc": "2026-09-24T01:43:58.811744+00:00",
"facility": "Qualcomm AI Hub Workbench",
"qai_hub_version": "0.55.0",
"source_compile_job_id": "j5mmy28w5",
"target_model_id": "mn0wyzd3q",
"input_dataset_id": "d7m80jl32",
"device": "Dragonwing RB3 Gen 2 Vision Kit",
"options": "--compute_unit cpu",
"kind": "inference",
"status": "SUBMITTED",
"job_id": "jglyxw4m5",
"url": "https://workbench.aihub.qualcomm.com/jobs/jglyxw4m5/"
}
@@ -24,6 +24,8 @@ A [MiniCPM-o speech-head study](../../../e2e_expansion_20260924/evidence/minicpm
A [raw sustained-window audit](../../../e2e_expansion_20260924/evidence/qwen_tts_sustained_drift/README.md) found that native Windows RTX Qwen3-TTS playback slows sharply after about 26 minutes: all 17 serial medium requests in minutes 27–30 underrun in the simulated first-audio playout, with median RTF 1.331. The separate WSL run stayed below real time without underruns. The cause is not established; a timestamped GPU telemetry repeat is underway.
Additional [Galaxy S24 and RB3 InternVLA Cosmos encoder checks](../../../e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md) returned finite same-fixture latents at 0.0613% and 2.20e-6 relative L2 versus source on S24 NPU-requested and RB3 CPU-requested routes, respectively. Actual placement profiles and complete action policies remain pending.
## Configuration × model
The WSL CPU Qwen3-TTS cell also has a separate bounded Omni whole-session
@@ -45,9 +47,9 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
| PC Snapdragon X Elite CRD / AI Hub | NOT E2E (one Spark attention layer on NPU) | NOT E2E (CPU talker/vocoder steps; QNN vocoder numeric) | NOT E2E | NOT E2E (NPU speech-head step profiled) | NOT E2E (Cosmos NPU encoder profiled) |
| Mobile Galaxy S25 / Snapdragon 8 Elite for Galaxy | NOT E2E (one Spark attention layer on NPU) | NOT E2E (FP16 talker token mismatch) | NOT E2E | NOT E2E (NPU speech head step profiled) | NOT E2E (Cosmos NPU encoder profiled) |
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | NOT E2E (one Spark attention layer on NPU) | NOT E2E (FP16 talker token mismatch) | NOT E2E | NOT E2E (NPU speech head step profiled) | NOT E2E |
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | NOT E2E (one Spark attention layer on NPU) | NOT E2E (FP16 talker token mismatch) | NOT E2E | NOT E2E (NPU speech head step profiled) | NOT E2E (Cosmos encoder numeric; placement pending) |
| Embedded SA8775P ADP / AI Hub | NOT E2E (one Spark attention layer on NPU) | NOT E2E (CPU talker/vocoder steps; NPU vocoder failed) | NOT E2E | NOT E2E (NPU speech-head step profiled) | NOT E2E (Cosmos NPU encoder profiled) |
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | NOT E2E (Spark CPU layer passed; W8A8 NPU quality open) | NOT E2E (mixed vocoder numeric unqualified) | NOT E2E | NOT E2E (CPU speech-head step profiled) | NOT E2E |
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | NOT E2E (Spark CPU layer passed; W8A8 NPU quality open) | NOT E2E (mixed vocoder numeric unqualified) | NOT E2E | NOT E2E (CPU speech-head step profiled) | NOT E2E (Cosmos CPU numeric; placement pending) |
## Per-cell reasons and next steps
@@ -102,7 +104,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | Qwen3-TTS 0.6B CustomVoice | TALKER_FP16_NUMERIC_FAIL; NOT_E2E_PROFILEABLE | The real 28-layer FP16 QNN talker step executed on the synthetic cache fixture, but changed CPU top-1 speech token 80 to 1720 with 9.33% logits relative L2 and maximum new-K/V relative L2 11.23%. This differs from an earlier FP16 decoder parity test (39.13 dB SNR) on a different component/fixture; the tested INT8 decoder parity failed with negative SNR. No full local TTS stream exists. | Resolve the [talker numerical difference](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md), then validate the complete local pipeline and a quality-passing artifact set. |
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | MiniCPM-o 4.5 | COMPONENT_FP16_SPEECH_HEAD_NPU_NUMERIC_PROFILED; NOT_E2E_PROFILEABLE | The retained FP16 speech-head job used a dataset with identical tensors to the recovered S25 fixture. Against original ONNX CPU, logits relative L2 was 0.3459%, maximum new-K/V relative L2 1.235%, and top-1 speech token 1867 matched. Earlier parity reported 49.22 dB SNR/top-1 match; tested W8A16 top-1 failed. The 100-sample component profile had p50/p95 8.537/9.006 ms, with all 1,029 execution-detail rows on NPU and reported inference peak memory 165,433,344 bytes. Generated-audio quality and the rest of the model remain unqualified. | Qualify thinker/encoders/vocoder and stateful local integration; [speech-head evidence](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md). |
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
| Mobile Galaxy S24 / Snapdragon 8 Gen 3 | InternVLA-A1 | COMPONENT_COSMOS_NUMERIC_MEASURED; PLACEMENT_PENDING; NOT_E2E_PROFILEABLE | The real Place_Markpen Cosmos FP16 QNN image encoder returned a finite six-frame synthetic-pattern latent on exact hosted S24 at 0.0613% relative L2 / 64.25 dB SNR versus pinned FP32 source. NPU-requested inference alone does not establish actual node placement; its profile is pending. No real-observation action tolerance, complete local policy or stateful controller exists. ADB has no attached device. | Audit the [S24 Cosmos placement profile](../../../e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md), then validate real observations, action units/timing, state, admission and cancellation on-device. |
| Embedded SA8775P ADP / AI Hub | Spark-X2.5 | COMPONENT_NPU_INFERENCE_PROFILED; NOT_E2E_PROFILEABLE | A real Spark-X2.5-1.7B one-layer W8A16 QDQ attention artifact, compiled as QNN DLC, ran on the exact hosted SA8775P ADP (Android 14). Inference returned finite hidden/new-K/new-V tensors bitwise identical to the prior S24/S25 NPU outputs on the same fixture; the profile attributed all 73 execution-detail rows to NPU. Across 100 component samples, nearest-rank p50/p95 was 1.837/2.079 ms; the 1.749 ms estimated-time field is the minimum. Hidden/K/V relative L2 versus original FP32 ONNX CPU was 1.351%/0.377%/0.650%, without a next-token quality tolerance. There is no complete 28-layer prefill/decode/KV-ring/sampling loop, resident Omni controller or complete-request timing. | Qualify a complete embedded resident artifact/controller with state, token quality, memory admission/cancel and sustained power; [SA8775P NPU raw evidence](../../../e2e_expansion_20260923/evidence/spark_sa8775_full_attention/README.md). |
| Embedded SA8775P ADP / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_CPU_VOCODER_PARITY_AND_PLACEMENT_PASS; TALKER_TFLITE_CPU_NUMERIC_PROFILED; QNN_NPU_VOCODER_INFERENCE_FAILED; COMPONENT_GPU_NUMERIC_FAIL; TALKER_FP16_NUMERIC_FAIL; NOT_E2E_PROFILEABLE | On the exact SA8775P, the same TFLite binary's CPU-requested Code2Wav inference matched original ONNX CPU within 4.13e-6 relative L2 / 107.67 dB SNR, without saturation. Its 100-sample CPU-requested component profile had p50/p95 5.384/5.534 s with all 738 execution-detail rows on CPU; the service estimated-time field 5.250 s is the sample minimum. The GPU-requested output saturated 96.63% of samples with 11.832 relative L2; its fresh 100-sample mixed CPU/GPU profile p50/p95 was 1.776/1.880 s, timing failed audio. Even with FP32-preserving GPU options, 84.89% of samples saturated with 11.389 relative L2. The 28-layer FP16 QNN talker step executed but changed CPU top-1 token 80 to 1720 with 9.43% logits relative L2. A separate TFLite CPU-requested talker step preserved CPU top-1 token 80 with logits relative L2 3.33e-5 and maximum K/V relative L2 3.41e-5. Its 100-sample component profile had p50/p95 52.635/76.925 ms with all 1,800 execution-detail rows on CPU. A separate FP16 QNN DLC vocoder compile passed, but same-fixture NPU-requested inference failed after compilation without a waveform or placement samples. No complete pipeline exists. | Isolate the FP16 QNN vocoder runtime failure, GPU delegate failure and FP16 talker numerical drift, then establish listening quality, co-resident streaming, memory and sustained behavior; [vocoder](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md) and [talker](../../../e2e_expansion_20260924/evidence/qwen_tts_talker_qualcomm/README.md) evidence. |
| Embedded SA8775P ADP / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
@@ -112,7 +114,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | Qwen3-TTS 0.6B CustomVoice | COMPONENT_MIXED_VOCODER_PROFILED_NUMERIC_UNQUALIFIED; NOT_E2E_PROFILEABLE | Fresh same-fixture TFLite GPU-requested vocoder inference returned a finite unsaturated waveform, but relative L2 versus original ONNX CPU was 15.73% (16.06 dB SNR), without a listening-quality tolerance; it is not a qualified audio path. The fresh 100-sample component profile had nearest-rank p50/p95 3.099/3.133 s and 403 CPU plus 335 GPU execution-detail rows. Its 3.070 s estimated-time field is the sample minimum, not median. The historical NPU talker build was rejected for floating-point inputs; no full model pipeline exists. | Establish waveform quality and a compatible integer talker artifact, then validate complete local execution, memory and sustained behavior; [vocoder evidence](../../../e2e_expansion_20260924/evidence/qwen_tts_vocoder_qualcomm/README.md). |
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | MiniCPM-o 4.5 | COMPONENT_SPEECH_HEAD_FP16_QNN_LOAD_FAILED; CPU_FALLBACK_NUMERIC_PROFILED; NOT_E2E_PROFILEABLE | The real 20-layer MiniCPM-o speech-head FP16 QNN DLC compiled for exact RB3 Gen 2, but same-fixture NPU-requested inference failed at graph load with QnnModel_composeGraphsFromDlc: MODEL_GRAPH_ERROR. RB3 device attributes do not advertise FP16 HTP support. An explicit TFLite CPU-requested fallback returned 41 finite tensors, preserved ONNX CPU top-1 token 1867, and had logits relative L2 1.26e-6 and maximum K/V relative L2 1.58e-6. Its 100-sample component profile had p50/p95 59.518/71.347 ms with all 1,008 execution-detail rows on CPU; reported inference peak memory was 820,981,760 bytes. No complete model bundle or local stateful controller exists. | Qualify complete multimodal generation and sustained behavior; [RB3 speech-head CPU evidence](../../../e2e_expansion_20260924/evidence/minicpmo_tts_qualcomm/README.md). |
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | InternVLA-A1 | NOT_E2E_PROFILEABLE | No complete qualified target artifact and device-local stateful Omni backend. Hub jobs provide component execution, not an on-device generation/streaming controller; ADB has no attached device. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
| Embedded RB3 Gen 2 / QCS6490 / AI Hub | InternVLA-A1 | COMPONENT_COSMOS_CPU_NUMERIC_MEASURED; PLACEMENT_PENDING; NOT_E2E_PROFILEABLE | The real Place_Markpen Cosmos ONNX CPU-requested image encoder returned a finite six-frame synthetic-pattern latent on exact hosted RB3 at 2.20e-6 relative L2 / 113.17 dB SNR versus pinned FP32 source. Actual CPU node placement and component timing await its profile; no complete local action policy or stateful controller exists. | Audit the [RB3 Cosmos CPU placement profile](../../../e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md), then validate real observations, action units/timing, state, admission and cancellation. |
## Measured request timing
+2
View File
@@ -27,6 +27,8 @@ The [native Windows Qwen3-TTS sustained audit](../../benchmarks/edge_harness/res
For M4, a [Galaxy S25 Cosmos encoder attempt](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/README.md) repackages the real Place_Markpen FP32 ONNX export after retaining two CPU parity fixtures. FLOAT16 QNN DLC inference returned a latent within 0.0524% relative L2 of the source on one synthetic pattern. Injecting that output into the real policy on local CPU changed synthetic actions by 0.742% relative L2, without a task tolerance. S25 six-frame component profile returned 100 samples at p50/p95 49.665/51.382 ms, with all 121 execution-detail rows on NPU; the action policy did not run on S25. The same encoder compiled to FP16 QNN DLC for X Elite CRD; same-fixture NPU-requested inference returned a finite latent at 0.0608% relative L2 versus source, with its 100-sample six-frame profile at p50/p95 82.939/83.837 ms and all 121 execution-detail rows on NPU. An SA8775P FP16 QNN DLC inference returned a finite same-fixture latent at 0.0613% relative L2 versus source; its 100-sample six-frame profile measured p50/p95 116.021/116.327 ms with all 121 execution-detail rows on NPU. This is component/action-sensitivity evidence, not device-local M4 qualification. The checkpoint train config names a2d_real/a2d_pen_holder; an [authenticated access check](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/internvla_real_observation_access/README.md) returned HTTP 404. The official vLLM-Omni example names an InternData-A1 Place_Markpen archive, but that dataset returned HTTP 403 gated-access denial under the current account, so real-observation reference-action quality is still unmeasured.
The same pinned Cosmos encoder now has [Galaxy S24 FP16 QNN](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/s24/audit_report.json) and [RB3 ONNX CPU](../../benchmarks/edge_harness/results/e2e_expansion_20260924/evidence/internvla_cosmos_qualcomm/rb3_cpu/audit_report.json) same-fixture numerical results at 0.0613% and 2.20e-6 relative L2 versus source. Their placement profiles are pending. Neither is a complete local policy or a real-observation task-quality result.
The [WSL CPU expansion](../../benchmarks/edge_harness/results/e2e_expansion_20260923/README.md)
adds constrained three-stage MiniCPM-o text-to-speech requests and a CPU-only
InternVLA synthetic policy forward. A subsequent 20-request serial MiniCPM-o