Record InternVLA FP32 AMD NPU compiler failure in matrix

Extract versioned stage contracts, model shared CPU/iGPU/NPU memory constraints, and add explicit graph routes, reservations, output acknowledgements, and worker retirement. Fix native Windows shared-buffer lifetime and host paths. Document qualified hardware paths and remaining rollout gates.

.buildkite/amd/scripts/bootstrap-amd-omni.sh

@@ -14,6 +14,8 @@ set -euo pipefail
 # shellcheck disable=SC1091
 source .buildkite/common/scripts/resolve_skip_ci.sh
 
+AMD_SUITE_SELECTOR=".buildkite/amd/scripts/select_test_suites.py"
+
 if [[ -z "${RUN_ALL:-}" ]]; then
     RUN_ALL=0
 fi
@@ -30,31 +32,20 @@ if [[ -z "${AMD_MIRROR_HW:-}" ]]; then
     AMD_MIRROR_HW="amdproduction"
 fi
 
-fail_fast() {
-    DISABLE_LABEL="ci-no-fail-fast"
-    # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
-    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
-        PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
-        if [[ $PR_LABELS == *"$DISABLE_LABEL"* ]]; then
-            echo false
-        else
-            echo true
-        fi
-    else
-        echo false  # not a PR or BUILDKITE_PULL_REQUEST not set
+PR_LABELS=""
+# Explicit debug suites do not depend on PR labels.
+if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -z "${DEBUG_TEST_YAML:-}" ]]; then
+    if ! PR_LABELS=$(curl -fsSL \
+        "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" \
+        | jq -r '.labels[].name'); then
+        echo "ERROR: Could not read PR labels; refusing to select an incorrect AMD test tier." >&2
+        exit 1
     fi
-}
+fi
 
-check_run_all_label() {
-    RUN_ALL_LABEL="ready-run-all-tests"
-    # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
-    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
-        PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
-        if [[ $PR_LABELS == *"$RUN_ALL_LABEL"* ]]; then
-            echo true
-        else
-            echo false
-        fi
+fail_fast() {
+    if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
+        echo true
     else
         echo false  # not a PR or BUILDKITE_PULL_REQUEST not set
     fi
@@ -64,6 +55,57 @@ if [[ -z "${COV_ENABLED:-}" ]]; then
     COV_ENABLED=0
 fi
 
+resolve_test_specs() {
+    local selected_specs
+    if ! selected_specs=$(python3 "$AMD_SUITE_SELECTOR" \
+        --branch "$BUILDKITE_BRANCH" \
+        --labels "$PR_LABELS" \
+        --debug-test-yaml "${DEBUG_TEST_YAML:-}"); then
+        exit 1
+    fi
+
+    TEST_SPECS=()
+    while IFS= read -r suite_spec; do
+        [[ -z "$suite_spec" ]] && continue
+        TEST_SPECS+=("$suite_spec")
+    done <<< "$selected_specs"
+}
+
+filter_test_specs_by_skip_ci() {
+    local suite_spec level decision
+    local skip_all=0
+    local -a runnable_specs=()
+
+    for suite_spec in "${TEST_SPECS[@]}"; do
+        case "$suite_spec" in
+            READY_TESTS:*) level="l2" ;;
+            MERGE_TESTS:*) level="l3" ;;
+            *)
+                echo "ERROR: unknown AMD test suite spec '$suite_spec'" >&2
+                exit 1
+                ;;
+        esac
+
+        if decision=$(python3 "$SKIP_CI_PY" gate amd "$level"); then
+            echo "Skipping AMD $level suite due to $decision decision."
+            [[ "$decision" == "skip-all" ]] && skip_all=1
+        else
+            runnable_specs+=("$suite_spec")
+        fi
+    done
+
+    TEST_SPECS=("${runnable_specs[@]}")
+    if [[ ${#TEST_SPECS[@]} -eq 0 ]]; then
+        if [[ $skip_all -eq 1 ]]; then
+            buildkite-agent annotate \
+                ":memo: CI skipped — docs or pytest skip-mark changes only" \
+                --style "info" 2>/dev/null || true
+        fi
+        echo "No AMD L2/L3 suites remain after skip-ci filtering."
+        exit 0
+    fi
+}
+
 upload_pipeline() {
     echo "Uploading pipeline..."
     # Install minijinja
@@ -84,7 +126,7 @@ upload_pipeline() {
     # (WIP) Use pipeline generator instead of jinja template
     if [ -e ".buildkite/amd/pipeline_generator/pipeline_generator.py" ]; then
         python -m pip install click pydantic
-        python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
+        python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all="$RUN_ALL" --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
         buildkite-agent pipeline upload .buildkite/amd/pipeline.yaml
         exit 0
     fi
@@ -97,49 +139,12 @@ upload_pipeline() {
 
     cd .buildkite/amd
 
-    # Select test definition file: merge suite for main, ready suite for PRs.
-    # For debugging, DEBUG_TEST_YAML accepts a comma-separated list containing
-    # "merge" and/or "ready" (case-insensitive). Multiple suites are combined
-    # before rendering so they share one amd-build step.
-    if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
-        declare -a DEBUG_TEST_SPECS=()
-        declare -A SEEN_DEBUG_TESTS=()
-        IFS=',' read -ra REQUESTED_DEBUG_TESTS <<< "${DEBUG_TEST_YAML,,}"
-
-        for requested_test in "${REQUESTED_DEBUG_TESTS[@]}"; do
-            # Permit readable values such as "ready, merge," and ignore the
-            # empty item produced by a trailing comma.
-            requested_test="${requested_test//[[:space:]]/}"
-            [[ -z "$requested_test" ]] && continue
-
-            if [[ -n "${SEEN_DEBUG_TESTS[$requested_test]:-}" ]]; then
-                echo "ERROR: duplicate DEBUG_TEST_YAML suite '$requested_test'" >&2
-                exit 1
-            fi
-            SEEN_DEBUG_TESTS[$requested_test]=1
-
-            case "$requested_test" in
-                ready)
-                    DEBUG_TEST_SPECS+=("READY_TESTS:test-amd-ready.yml")
-                    ;;
-                merge)
-                    DEBUG_TEST_SPECS+=("MERGE_TESTS:test-amd-merge.yml")
-                    ;;
-                *)
-                    echo "ERROR: DEBUG_TEST_YAML entries must be 'merge' or 'ready', got '$requested_test'" >&2
-                    exit 1
-                    ;;
-            esac
-        done
-
-        if [[ ${#DEBUG_TEST_SPECS[@]} -eq 0 ]]; then
-            echo "ERROR: DEBUG_TEST_YAML did not contain a test suite" >&2
-            exit 1
-        elif [[ ${#DEBUG_TEST_SPECS[@]} -eq 1 ]]; then
-            TEST_YAML="${DEBUG_TEST_SPECS[0]#*:}"
-        else
-            TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-debug-tests.XXXXXX.yml")
-            python - "$TEST_YAML" "${DEBUG_TEST_SPECS[@]}" <<'PY'
+    # Multiple label-selected or debug-selected suites share one image build.
+    if [[ ${#TEST_SPECS[@]} -eq 1 ]]; then
+        TEST_YAML="${TEST_SPECS[0]#*:}"
+    else
+        TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-selected-tests.XXXXXX.yml")
+        python - "$TEST_YAML" "${TEST_SPECS[@]}" <<'PY'
 import sys
 
 import yaml
@@ -172,13 +177,8 @@ for suite_spec in suite_specs:
 with open(output_path, "w", encoding="utf-8") as output_file:
     yaml.safe_dump(combined, output_file, sort_keys=False)
 PY
-        fi
-        echo "DEBUG_TEST_YAML override: using ${DEBUG_TEST_SPECS[*]}"
-    elif [[ $BUILDKITE_BRANCH == "main" ]]; then
-        TEST_YAML="test-amd-merge.yml"
-    else
-        TEST_YAML="test-amd-ready.yml"
     fi
+    echo "AMD test suites: ${TEST_SPECS[*]}"
 
     (
         set -x
@@ -198,7 +198,7 @@ PY
             > pipeline.yaml
     )
     cat pipeline.yaml
-    if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-debug-tests."*.yml ]]; then
+    if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-selected-tests."*.yml ]]; then
         rm -f -- "$TEST_YAML"
     fi
     buildkite-agent artifact upload pipeline.yaml
@@ -219,12 +219,10 @@ if [[ $BUILDKITE_BRANCH == "main" ]]; then
     file_diff=$(get_diff_main)
 fi
 
-# Early exit: unified skip-ci (docs / skip marks) and CI-yaml-only level targeting.
-if [[ $BUILDKITE_BRANCH == "main" ]]; then
-    gate_bootstrap_ci amd l3
-else
-    gate_bootstrap_ci amd l2
-fi
+# Resolve PR tier labels before skip-ci so ready and merge suites can be
+# filtered independently for CI-YAML-only changes.
+resolve_test_specs
+filter_test_specs_by_skip_ci
 
 patterns=(
     "docker/Dockerfile"
@@ -274,14 +272,6 @@ for file in $file_diff; do
     fi
 done
 
-# Check for ready-run-all-tests label
-LABEL_RUN_ALL=$(check_run_all_label)
-if [[ $LABEL_RUN_ALL == true ]]; then
-    RUN_ALL=1
-    NIGHTLY=1
-    echo "Found 'ready-run-all-tests' label. Running all tests including optional tests."
-fi
-
 # Decide whether to use precompiled wheels
 # Relies on existing patterns array as a basis.
 if [[ -n "${VLLM_USE_PRECOMPILED:-}" ]]; then

.buildkite/amd/scripts/select_test_suites.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Select AMD L2/L3 suites after Buildkite has created an AMD build.
+
+This repository-side selector does not decide which GitHub label events start
+the external ``vllm-omni-amd-ci`` pipeline. Its Buildkite PR build condition
+must admit the relevant label event before this code can inspect the labels.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Iterable
+
+SUITE_SPECS = {
+    "ready": "READY_TESTS:test-amd-ready.yml",
+    "merge": "MERGE_TESTS:test-amd-merge.yml",
+}
+
+
+def _parse_debug_suites(value: str) -> tuple[str, ...]:
+    suites = tuple(item.strip().lower() for item in value.split(",") if item.strip())
+    if not suites:
+        raise ValueError("DEBUG_TEST_YAML did not contain a test suite")
+
+    seen: set[str] = set()
+    for suite in suites:
+        if suite not in SUITE_SPECS:
+            raise ValueError(f"DEBUG_TEST_YAML entries must be 'merge' or 'ready', got {suite!r}")
+        if suite in seen:
+            raise ValueError(f"duplicate DEBUG_TEST_YAML suite {suite!r}")
+        seen.add(suite)
+    return suites
+
+
+def select_amd_test_suites(
+    *,
+    branch: str,
+    labels: Iterable[str],
+    debug_test_yaml: str = "",
+) -> tuple[str, ...]:
+    """Return ordered suite names while preserving legacy unlabeled PRs."""
+
+    if debug_test_yaml:
+        return _parse_debug_suites(debug_test_yaml)
+    if branch == "main":
+        return ("merge",)
+
+    label_set = {label.strip() for label in labels if label.strip()}
+    selected = tuple(suite for label, suite in (("ready", "ready"), ("merge-test", "merge")) if label in label_set)
+    # AMD historically uploaded L2 for every PR build. Keep that behavior for
+    # triggers such as amd-test while adding tier selection for ready/merge-test.
+    return selected or ("ready",)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--branch", required=True)
+    parser.add_argument("--labels", default="")
+    parser.add_argument("--debug-test-yaml", default="")
+    args = parser.parse_args()
+
+    labels = tuple(args.labels.splitlines())
+    try:
+        suites = select_amd_test_suites(
+            branch=args.branch,
+            labels=labels,
+            debug_test_yaml=args.debug_test_yaml,
+        )
+    except ValueError as exc:
+        parser.error(str(exc))
+
+    label_set = {label.strip() for label in labels if label.strip()}
+    if not args.debug_test_yaml and args.branch != "main" and not ({"ready", "merge-test"} & label_set):
+        print(
+            "No AMD L2/L3 tier label found; preserving the legacy ready-suite fallback.",
+            file=sys.stderr,
+        )
+    if "nightly-test" in label_set:
+        print(
+            "AMD nightly-test has no suite yet and does not affect L2/L3 selection.",
+            file=sys.stderr,
+        )
+
+    for suite in suites:
+        print(SUITE_SPECS[suite])
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.buildkite/amd/test-amd-ready.yml

@@ -107,6 +107,11 @@ steps:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
         - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
+        - >-
+          timeout 20m pytest -s -v
+          tests/diffusion/models/ltx2/test_ltx2_transformer_ulysses.py
+          -m 'core_model and rocm and cards_2'
+          --run-level "core_model"
 
     - label: "Diffusion GPU Worker Test"
       agent_pool: mi300_2
@@ -122,7 +127,13 @@ steps:
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
-        - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
+        # Preserve legacy one-GPU tests without cards_1 while excluding every
+        # registered multi-GPU marker from this one-GPU job.
+        - >-
+          timeout 15m pytest -sv tests/diffusion/models/
+          -m 'core_model and cuda and not
+          (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
+          --run-level "core_model"
 
     - label: "Diffusion · Wan22 Test"
       agent_pool: mi300_1

.buildkite/common/ci_mirror_hardwares.yml

@@ -17,6 +17,8 @@
 #   a2b3_npu_1  — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
 #   a2b3_npu_4  — ascend-a2b3, resource_class npu-4
 #   a3_npu_2    — ascend-a3, resource_class npu-2 (A3 CI image tag …:$BUILDKITE_COMMIT-a3)
+#   a5_npu_2    — ascend-a5, resource_class npu-2
+#   310p_npu_1  — ascend-310p, resource_class npu-1
 #
 # B200 (b200-k8s): matches vLLM device ``b200-k8s`` from ci-infra
 # (AgentQueue.B200_K8S + b200_plugin_template). Queue routes the K8s pool;
@@ -26,6 +28,9 @@
 # packed onto g6.12xlarge L4 nodes (``vllm.ci/gpu-pool=l4x4``). GPU / CPU /
 # memory / shm requests scale with card count (1 GPU = 10 CPU / 40Gi / 24Gi shm).
 # Retry covers lost agents / killed pods, not pytest or OOM (exit 1 / 137).
+# The Kubernetes HF secret deliberately uses VLLM_CI_HF_TOKEN. Scheduled
+# build-level HF_TOKEN values can override pod env entries with the same name;
+# upload_pipeline.py restores the secret-backed value inside the job shell.
 
 x-npu-k8s-plugins: &npu_k8s_plugins
   - kubernetes:
@@ -84,7 +89,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /fsx/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -135,7 +140,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /fsx/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -186,7 +191,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /fsx/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -237,7 +242,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /fsx/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -271,7 +276,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -306,7 +311,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -341,7 +346,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -376,7 +381,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -426,7 +431,7 @@ mirror_hardwares:
                     value: "0"
                   - name: HF_HOME
                     value: /raid/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -477,7 +482,7 @@ mirror_hardwares:
                     value: "0"
                   - name: HF_HOME
                     value: /raid/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -528,7 +533,7 @@ mirror_hardwares:
                     value: "0"
                   - name: HF_HOME
                     value: /raid/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -579,7 +584,7 @@ mirror_hardwares:
                     value: "0"
                   - name: HF_HOME
                     value: /raid/hf_cache
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -646,3 +651,52 @@ mirror_hardwares:
       resource_class: "npu-16"
     image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
     plugins: *npu_k8s_plugins
+
+  a5_npu_2:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-2"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  a5_npu_4:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  a5_npu_8:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-8"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  a5_npu_16:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-16"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  310p_npu_1:
+    agents:
+      queue: "ascend-310p"
+      resource_class: "npu-1"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-310p"
+    plugins: *npu_k8s_plugins
+
+  310p_npu_2:
+    agents:
+      queue: "ascend-310p"
+      resource_class: "npu-2"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-310p"
+    plugins: *npu_k8s_plugins
+
+  310p_npu_4:
+    agents:
+      queue: "ascend-310p"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-310p"
+    plugins: *npu_k8s_plugins

.buildkite/common/scripts/upload_pipeline.py

@@ -70,7 +70,15 @@ from tests.helpers.mark import (  # noqa: E402
 
 LOG = "upload_pipeline"
 BOOTSTRAP_STEPS_FILENAME = "bootstrap-upload-steps.yml"
-BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset({"image-build", "image-build-a2", "image-build-a3"})
+BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset(
+    {
+        "image-build",
+        "image-build-a2",
+        "image-build-a3",
+        "image-build-a5",
+        "image-build-310p",
+    }
+)
 BOOTSTRAP_UPLOAD_IF_KEYS = {
     "upload-ready-pipeline": "ready",
     "upload-merge-pipeline": "merge",
@@ -79,6 +87,8 @@ BOOTSTRAP_UPLOAD_IF_KEYS = {
 }
 E2E_GROUP_MARKER = "E2E Test"
 CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
+CUDA_HF_TOKEN_ENV = "VLLM_CI_HF_TOKEN"
+CUDA_HF_TOKEN_EXPORT = f'if [ -n "$${{{CUDA_HF_TOKEN_ENV}:-}}" ]; then export HF_TOKEN="$${{{CUDA_HF_TOKEN_ENV}}}"; fi'
 
 # Bootstrap Buildkite ``if`` expressions.
 # ``*_MAIN_IF``: main + env schedule. ``*_LABEL_IF``: PR label (and/or composed with MAIN).
@@ -501,6 +511,17 @@ def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None:
 
     expanded = copy.deepcopy(preset)
     merged = {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
+    # A scheduled build's HF_TOKEN may override a pod env entry with the same
+    # name. The preset injects the Kubernetes secret under a collision-proof
+    # alias; restore the conventional name after Buildkite has merged env.
+    if any(preset_name == chip or preset_name.startswith(f"{chip}_") for chip in _cuda_mirror_chips()):
+        commands = merged.get("commands")
+        if isinstance(commands, list):
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, *commands]
+        elif commands is None:
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT]
+        else:
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, commands]
     # Preset retry (K8S_RETRY on l4_*) must not clobber a step that opted out.
     if "retry" in step:
         merged["retry"] = step["retry"]

.buildkite/cuda/test-merge.yml

@@ -79,6 +79,8 @@ steps:
     steps:
       - label: "TTS · Qwen3-TTS CustomVoice Test"
         source_file_dependencies:
+          - tests/e2e/features/prefill_phase/
+          - vllm_omni/worker/gpu_model_runner.py
           - tests/helpers/media.py
           - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
           - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
@@ -96,7 +98,7 @@ steps:
             timeout 40m bash -c "
               export VLLM_LOGGING_LEVEL=DEBUG
               export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py tests/e2e/features/prefill_phase/test_chunked_prefill_tail.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
         mirror_hardwares: l4_1
 
@@ -153,6 +155,7 @@ steps:
         source_file_dependencies:
           - tests/e2e/offline_inference/test_minicpmo_4_5.py
           - tests/e2e/online_serving/test_minicpmo_4_5.py
+          - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja
           - vllm_omni/model_executor/models/minicpmo_4_5/
           - vllm_omni/model_executor/models/step_audio2/
           - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/

.buildkite/cuda/test-ready.yml

@@ -76,7 +76,7 @@ steps:
 
       - label: "Diffusion · Other Test"
         commands:
-          - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
+          - timeout 15m pytest -sv tests/diffusion/*.py tests/diffusion/ar_diffusion -m 'core_model and cuda' --run-level "core_model"
         mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
@@ -93,7 +93,7 @@ steps:
     commands:
       - |
         timeout 15m bash -c "
-                  pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
+                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py -m 'core_model and cuda' --run-level 'core_model'
         "
     mirror_hardwares: l4_1
 
@@ -188,6 +188,7 @@ steps:
       - label: "Omni · MiniCPM-o 4.5 Online Test"
         source_file_dependencies:
           - tests/e2e/online_serving/test_minicpmo_4_5.py
+          - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja
           - vllm_omni/model_executor/models/minicpmo_4_5/
           - vllm_omni/model_executor/models/step_audio2/
           - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/

.buildkite/npu/bootstrap-upload-steps.yml

@@ -3,10 +3,10 @@
 steps:
   - label: ":buildkit: Build and Push NPU Test Image - A2"
     key: image-build-a2
-    agents:
+    agents: &npu-image-build-agents
       queue: "ascend-a2b3"
       resource_class: "npu-1"
-    plugins:
+    plugins: &npu-image-build-plugins
       - kubernetes:
           metadata:
             annotations:
@@ -46,123 +46,47 @@ steps:
       IMAGE_NAME: "vllm-omni-ci-npu"
       IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
       BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
-
-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
-
-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}"
 
   - label: ":buildkit: Build and Push NPU Test Image - A3"
     key: image-build-a3
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    plugins:
-      - kubernetes:
-          metadata:
-            annotations:
-              vault.hashicorp.com/agent-init-first: "true"
-              vault.hashicorp.com/agent-inject: "true"
-              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
-              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
-              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
-              vault.hashicorp.com/agent-pre-populate-only: "true"
-              vault.hashicorp.com/agent-run-as-group: "1000"
-              vault.hashicorp.com/agent-run-as-user: "1000"
-              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
-              vault.hashicorp.com/role: ascend-gha-runners
-              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
-              vault.hashicorp.com/tls-skip-verify: "true"
-          podSpecPatch:
-            volumes:
-            - name: token-vol
-              projected:
-                defaultMode: 420
-                sources:
-                - serviceAccountToken:
-                    audience: api
-                    expirationSeconds: 600
-                    path: token
+    agents: *npu-image-build-agents
+    plugins: *npu-image-build-plugins
     env:
       VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3"
       IMAGE_NAME: "vllm-omni-ci-npu"
       IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
       BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}" -a3
 
-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
+  - label: ":buildkit: Build and Push NPU Test Image - A5"
+    key: image-build-a5
+    agents: *npu-image-build-agents
+    plugins: *npu-image-build-plugins
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a5"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}" -a5
 
-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci.a3 \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+  - label: ":buildkit: Build and Push NPU Test Image - 310P"
+    key: image-build-310p
+    agents: *npu-image-build-agents
+    plugins: *npu-image-build-plugins
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-310p"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}" -310p
 
   - label: "Upload Ready Pipeline"
     depends_on:
       - image-build-a2
       - image-build-a3
+      - image-build-a5
+      - image-build-310p
     key: upload-ready-pipeline
     agents:
       queue: "cpu_queue_premerge"
@@ -174,6 +98,8 @@ steps:
     depends_on:
       - image-build-a2
       - image-build-a3
+      - image-build-a5
+      - image-build-310p
     key: upload-nightly-pipeline
     commands:
       - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-nightly.yml

.buildkite/npu/scripts/build-npu-image.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -ex
+
+dockerfile=${1:?Dockerfile is required}
+image_tag=${2:?Image tag is required}
+device_suffix=${3:-}
+
+image_name=${IMAGE_NAME:-vllm-omni-ci-npu}
+image_registry=${IMAGE_REGISTRY:-swr.cn-southwest-2.myhuaweicloud.com/modelfoundry}
+buildkitd_addr=${BUILDKITD_ADDR:-tcp://buildkitd-service.buildkitd:1234}
+
+echo "--- Building and pushing NPU Test Image"
+echo "Image: ${image_registry}/${image_name}:${image_tag}"
+echo "Dockerfile: ${dockerfile}"
+
+if ! command -v buildctl &> /dev/null; then
+  mkdir -p /tmp/buildkit
+  buildkit_version=v0.29.0
+  wget -q \
+    "https://github.com/moby/buildkit/releases/download/${buildkit_version}/buildkit-${buildkit_version}.linux-arm64.tar.gz" \
+    -O /tmp/buildkit.tar.gz
+  tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+  cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+fi
+
+build_args=()
+if [[ -n "${device_suffix}" ]]; then
+  build_args+=(--opt "build-arg:VLLM_ASCEND_DEVICE_SUFFIX=${device_suffix}")
+fi
+
+export DOCKER_CONFIG=/home/user/.docker
+buildctl \
+  --addr="${buildkitd_addr}" \
+  --tlscacert=/home/user/.docker/ca.pem \
+  --tlscert=/home/user/.docker/cert.pem \
+  --tlskey=/home/user/.docker/key.pem \
+  build \
+  --frontend dockerfile.v0 \
+  --local context=. \
+  --local dockerfile=./docker \
+  --opt "filename=${dockerfile}" \
+  "${build_args[@]}" \
+  --secret id=dockerconfig,src=/home/user/.docker/config.json \
+  --output "type=image,name=${image_registry}/${image_name}:${image_tag},push=true" \
+  --progress=plain
+
+echo "--- Image pushed successfully"
+echo "${image_registry}/${image_name}:${image_tag}"

.buildkite/npu/test-npu-ready.yml

@@ -1,26 +1,59 @@
 steps:
-  - label: "Platforms Test"
-    key: ready-platforms-test
+  - group: ":card_index_dividers: Simple Test"
     depends_on: upload-ready-pipeline
-    mirror_hardwares: a3_npu_2
-    commands:
-      - >-
-        pytest -s -v
-        tests/platforms/npu/test_minicpmo_code2wav_npugraph.py
-        -m 'core_model and npu and A3 and omni'
-        --run-level 'core_model'
+    steps:
+      - label: "Platforms Test"
+        key: ready-platforms-test
+        mirror_hardwares: a3_npu_2
+        commands:
+          - >-
+            pytest -s -v
+            tests/platforms/npu/test_minicpmo_code2wav_npugraph.py
+            -m 'core_model and npu and A3 and omni'
+            --run-level 'core_model'
 
-  - label: "Diffusion NPU layer tests"
-    key: ready-diffusion-npu-layer-tests
-    depends_on: upload-ready-pipeline
-    mirror_hardwares: a3_npu_2
-    commands:
-      - >-
-        pytest -s -v
-        tests/diffusion/layers/test_activation.py
-        tests/diffusion/layers/test_fused_qk_norm_rope_npu.py
-        -m 'core_model and npu and A3'
-        --run-level 'core_model'
+      - label: "Diffusion NPU layer tests - A2"
+        key: ready-diffusion-npu-layer-tests-a2
+        mirror_hardwares: a2b3_npu_1
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            -m 'core_model and npu and A2'
+            --run-level 'core_model'
+
+      - label: "Diffusion NPU layer tests - A3"
+        key: ready-diffusion-npu-layer-tests
+        mirror_hardwares: a3_npu_2
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            tests/diffusion/layers/test_fused_qk_norm_rope_npu.py
+            tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_swiglu.py
+            tests/diffusion/models/minimax_h3/test_minimax_h3_modulation.py
+            -m 'core_model and npu and A3'
+            --run-level 'core_model'
+
+      - label: "Diffusion NPU layer tests - A5"
+        key: ready-diffusion-npu-layer-tests-a5
+        mirror_hardwares: a5_npu_2
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            -m 'core_model and npu and A5'
+            --run-level 'core_model'
+
+      - label: "Diffusion NPU layer tests - 310P"
+        key: ready-diffusion-npu-layer-tests-310p
+        mirror_hardwares: 310p_npu_1
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            -m 'core_model and npu and 310P'
+            --run-level 'core_model'
 
   - group: ":card_index_dividers: Omni Model Test"
     key: ready-omni-group

.claude/skills/review-pr/SKILL.md

@@ -123,10 +123,11 @@ it changes again, report the churn and wait for a stable target.
 
 For a trusted PR head, materialize the pinned head in an isolated detached
 worktree. A worktree freezes identity but is not a security sandbox. Treat fork
-heads as untrusted unless the user and environment policy explicitly establish
-otherwise: execute them only in a disposable, secret-free sandbox with restricted
-filesystem, network, and resources; without one, use static SHA-addressed reads
-and CI evidence only. For a local review, freeze the committed, index, worktree,
+heads as untrusted until the user and environment policy explicitly establish
+trust: execute their code only after that trust is recorded for this host,
+with the trusted SHA noted in the review state and secrets kept out of the
+execution scope; without recorded trust, use static SHA-addressed reads and CI
+evidence only. For a local review, freeze the committed, index, worktree,
 and NUL-safe in-scope untracked contents. Follow
 [review-execution.md](references/process/review-execution.md) for trust gates,
 state fingerprints, and byte-for-byte staleness checks.
@@ -196,7 +197,7 @@ style preferences as simplification findings.
 
 Before each validation group, verify the frozen SHA plus the tracked, index,
 untracked, and ignored-file fingerprint, or recreate a pristine snapshot. On a
-trusted head or inside the required sandbox, run an import/version preflight,
+head the user explicitly trusted for this host, run an import/version preflight,
 then the narrowest relevant tests and low-cost static checks. Bind every result
 to the head SHA, snapshot fingerprint, and environment fingerprint. Never run
 imports, tests, builds, hooks, or repo-configurable tooling from an untrusted

.claude/skills/review-pr/references/checks/perf-verification.md

@@ -22,6 +22,19 @@ Keep exact commands and report variability rather than a cherry-picked run.
 Use the repository's benchmark when it covers the claim; otherwise use the
 smallest reproducible workload that reaches the changed production path.
 
+## Isolation per claim
+
+Isolate one claim per comparison. When a PR stacks several changes (a fused or
+restructured load path, a distilled step schedule, a new cache), request one
+column per claim at equal steps/workload instead of one blended base-vs-head
+delta, and name the unchanged stage or metric that pins equivalence.
+
+When the changed path is runtime-switchable (feature flag or environment
+variable), run every column on the same head and toggle the switch rather than
+diffing against a base commit that cannot separate the stacked changes:
+base, head with the switch off, head with the switch on, then head with the
+switch on plus each further claim.
+
 ## Evidence table
 
 | Dimension | Typical evidence |
@@ -34,6 +47,100 @@ smallest reproducible workload that reaches the changed production path.
 Every optimization needs correctness or quality evidence. A faster result on a
 different workload, precision, seed, or topology is not a valid comparison.
 
+## E2e stage-attribution table
+
+For an end-to-end latency claim, request the stage-attribution table the
+repository's perf PRs use:
+
+- Rows carry the benchmark's own field names: total e2e, per-stage execution,
+  postprocess, response encoding, and a request-to-output roll-up. Add a stage
+  handoff/idle row when the pipeline spans stages — connector transfer and
+  scheduling gaps are invisible in per-stage latency deltas.
+- Sanity rows pin comparability: inference steps, output frames/pixels/samples,
+  and generated tokens held identical across columns.
+- Report P50 and P100 from the stated warmup and measured counts. One
+  population per number: never mix cold-start and warm samples into one value.
+- Name the baseline and candidate commit SHAs, and close with an isolation
+  statement naming the metric that stayed unchanged ("X remained effectively
+  unchanged, isolating the gain to Y").
+
+When the serving path caches, captures, or regenerates at runtime (CUDA-graph
+bucket growth, cache rebuilds, recompilation), medians hide the resulting tail.
+Also request the event count and the maximum observed stall.
+
+Component-level tables pair with, not replace, the e2e table: per-op reference
+vs candidate timings (with the numerical-equality result per op) plus one
+complete-pipeline check under the production shapes.
+
+## Proportionality
+
+When a PR adds a model and explicitly labels its numbers as smoke observations
+rather than claims, do not demand the attribution machinery. Ask only for
+evidence protecting the model's stated operating contract — for a
+realtime/duplex model that is sustained cadence over a realistic session
+(enough ticks to leave warmup), with any mid-session capture/regeneration
+event and its stall reported, since a cadence that collapses mid-session is a
+functional failure, not a perf claim.
+
+## Realtime/duplex world models
+
+Request the operating-contract evidence as serving metrics, not claims:
+
+- Sustained per-tick latency over one realistic session: median, max, and the
+  argmax tick.
+- Stall events with their causes.
+- Peak memory at first versus last tick — session-owned state must not grow
+  per tick.
+- Control latency from a submitted action to the tick that applies it.
+- Realtime-versus-offline parity under the same controls, using the
+  repository's video-similarity helpers. Parity plotted over tick index is the
+  drift-over-horizon curve and the honest measure of session lifetime;
+  distribution-level suites (FVD, VBench) belong to model evaluation, not the
+  serving integration.
+
+Watch the contract boundaries the model documents: if a control input is
+tick-API-only, the parity pair must use a horizon both paths can run.
+
+## Speech-generation (TTS) models
+
+The streaming contract outranks the average — a synthesis that pauses
+mid-stream is broken for realtime use regardless of its averages. Request:
+
+- The benchmark's own speech metrics over one long utterance (enough audio to
+  leave warmup, tens of seconds): `audio_ttfp` (time to first packet),
+  `audio_rtf` (real-time factor; above 1.0 cannot keep up), `e2el`, and
+  `ttft`/`tpot` where tokens precede audio.
+- Mid-utterance stalls as the benchmark's continuity fields, not prose:
+  `audio_underrun` percentiles and the `audio_continuity_ok` rate
+  (`vllm_omni/benchmarks/metrics/metrics.py`), computed by
+  `vllm_omni/benchmarks/audio_continuity.py` — the companion to `audio_ttfp`
+  and `audio_rtf`. A synthesis that bursts chunks fast enough on average but
+  underruns mid-stream fails the streaming contract even when the averages
+  pass.
+- Quality as a paired comparison against the reference implementation under
+  the same text, speaker prompt, and seed; report sample rate and frame
+  count, and assert output completeness — truncated or dropped trailing audio
+  is a known failure class, not a quality nuance.
+- For realtime or duplex TTS: interruption (barge-in) latency and the
+  per-session memory trend across turns.
+
+## Omni models
+
+Apply the e2e stage-attribution table directly to the voice-to-voice path of
+omni (speech-to-speech and multimodal AR) models:
+
+- TTFT for text output, `audio_ttfp` for audio output, and per-stage
+  ITL/TPOT cadence across the perception-understanding-generation chain.
+- Cross-modal parity — the omni analogue of realtime-versus-offline parity:
+  the same semantic request through text and audio inputs, compared on output
+  content or timing.
+- For duplex omni sessions: turn-taking latency, interruption handling, the
+  concurrent session count actually validated, and a per-turn memory trend —
+  session-owned KV state must not grow per turn. When either turn produces
+  audio, continuity is judged with the same `audio_underrun` and
+  `audio_continuity_ok` fields as TTS, since turn-taking latency alone does
+  not expose mid-utterance underruns.
+
 ## Reviewer verification ladder
 
 1. Run base/head A/B on suitable hardware when affordable.

.claude/skills/review-pr/references/process/general-checks.md

@@ -121,6 +121,19 @@ Search bounded callers and sibling implementations before reporting dead code,
 duplication, or a missed consumer. Keep new abstractions only when they have a
 distinct live caller, invariant, or compatibility need.
 
+Verify absence and equality claims with whole-file evidence. A line window, a
+grep for one symbol, or a commit message proves presence, never absence: to
+assert that a file matches another version or lacks a change, diff or hash the
+full files (for example `git diff <sha1>:<path> <sha2>:<path>` or the contents
+API's size and SHA fields) before anchoring the finding. A verification
+shortcut that would itself be wrong to run is not evidence.
+
+For a value written or finalized in more than one place, establish which site
+executes first on each entry path — and whether the earlier site makes the
+later ones unreachable — before naming where a fix belongs. Enumerating all
+call sites correctly still misses the bug when a path the enumeration ranked
+last actually finalizes first and clears the state the proposed fix checks.
+
 ## Finding bar
 
 Anchor each finding to a changed `path:line`; name the trigger or call path,

.claude/skills/review-pr/references/process/review-execution.md

@@ -39,13 +39,13 @@ isolated detached worktree. Run every source read, `rg` search, import, and test
 from that snapshot, not the caller's checkout. A detached worktree provides
 snapshot isolation, not a security boundary.
 
-Treat a fork head as untrusted unless the user and environment policy explicitly
-establish trust. Never run its imports, tests, builds, hooks, package setup, or
-repo-configurable linters/plugins on the reviewer host. Execute them only in a
-disposable sandbox or VM with no credentials or inherited secrets, no host agent
-or service sockets, minimal read-only host mounts, disabled or explicitly
-allowlisted network, resource/time limits, and destruction after the run. If
-that boundary is unavailable, limit the review to the remote diff, SHA-addressed
+Treat a fork head as untrusted until the user and environment policy
+explicitly establish trust. Without recorded trust, never run its imports,
+tests, builds, hooks, package setup, or repo-configurable linters/plugins on
+the reviewer host. Once the user explicitly trusts this pinned commit for
+this host, record the trusted SHA with the review state and execute with
+credentials, agent sockets, and other secrets out of scope. Without recorded
+trust, limit the review to the remote diff, SHA-addressed
 reads such as `git show <head_sha>:<path>`, and existing CI evidence; report all
 executable validation as a gap.
 

.gitignore

@@ -281,3 +281,20 @@ lance_compare/
 
 # Linting binaries downloaded by local pre-commit hooks
 shellcheck*/
+
+# --- edge-infer -------------------------------------------------------------
+# Measurement artifacts are the substance of this fork, not build output: every
+# published figure is traceable to a file below. They are tracked despite the
+# blanket `*.json` and `results/` rules above. Binary captures (.npz, .onnx,
+# .wav, .npy) stay ignored.
+!benchmarks/edge_harness/results/
+!benchmarks/edge_harness/results/**
+!baselines/
+!baselines/**
+benchmarks/edge_harness/results/**/*.npz
+benchmarks/edge_harness/results/**/*.onnx
+benchmarks/edge_harness/results/**/*.npy
+benchmarks/edge_harness/results/**/*.wav
+
+# Portable wire fixtures are source inputs, not generated measurements.
+!packages/omni-stage-contracts/conformance/*.json

CONTRIBUTING.md

@@ -2,6 +2,8 @@
 
 You may find information about contributing to vLLM-Omni on [Contributing](https://vllm-omni.readthedocs.io/en/latest/contributing/).
 
+AI-assisted contributions must follow the [AI Usage Policy](docs/contributing/README.md#ai-usage-policy), including disclosure, human review, and validation requirements.
+
 Local `pre-commit` gates (SPDX, forbidden imports, `torch.cuda`, shellcheck,
 mypy, test marks, TTS adapter ratchet, markdownlint, Buildkite schema, and
 how to extend allowlists) are documented in the

NOTICE

@@ -0,0 +1,34 @@
+edge-infer
+Copyright 2026 tzhouam
+
+This product is a derivative work of vLLM-Omni
+(https://github.com/vllm-project/vllm-omni), licensed under the Apache License,
+Version 2.0. Upstream code is Copyright the vLLM-Omni contributors, and vLLM-Omni
+is itself a derivative of vLLM (https://github.com/vllm-project/vllm).
+
+Modifications in this fork are confined to edge/CPU inference and are listed in
+README.md; the substantive ones are:
+
+  vllm_omni/model_executor/layers/quantization/cpu_int4.py
+      4-bit weight-only quantization for the CPU backend
+  vllm_omni/model_executor/models/spark2_5/spark2_5.py
+      RoPE table sized to max_model_len rather than the advertised context
+  vllm_omni/edge/
+      device-class weight-path selection, KV budget sizing, memory release
+  vllm_omni/platforms/cpu/, vllm_omni/patch.py
+      CPU platform and CPU-backend patches
+  benchmarks/edge_harness/
+      measurement tooling and results (original to this fork)
+  baselines/, docs/engines/, docs/analysis/
+      benchmark results and written analyses (original to this fork)
+
+vLLM itself is not modified and no vLLM source is redistributed here. This fork
+changes vLLM's CPU behaviour through its own extension points at runtime
+(vllm_omni/patch.py); the pinned upstream versions are recorded in
+docs/edge/environment.md.
+
+Baseline measurements reference llama.cpp (https://github.com/ggml-org/llama.cpp,
+MIT). No llama.cpp source is redistributed; only benchmark output produced with
+it, under baselines/llama_cpp/.
+
+Upstream's README is preserved at docs/README.vllm-omni.md.

README.md

@@ -1,109 +1,116 @@
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/logos/vllm-omni-logo.png">
-    <img alt="vllm-omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/logos/vllm-omni-logo.png" width=55%>
-  </picture>
-</p>
-<h3 align="center">
-Easy, fast, and cheap omni-modality model serving for everyone
-</h3>
+# edge-infer
 
-<p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://deepwiki.com/vllm-project/vllm-omni"><b>DeepWiki</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/1aPj0OGl_-ZVoib-Qne5dGDAlrRFB-PdHl6E-EE99g8E/edit?usp=sharing"><b>Slides</b></a> |
-</p>
+Running multimodal models on **edge CPUs** — a mini-PC or a gateway box with
+tens of gigabytes of RAM and no GPU — instead of on servers.
 
----
+Built as a fork of [vLLM-Omni](https://github.com/vllm-project/vllm-omni),
+because it is the only engine with a multi-stage multimodal pipeline,
+continuous batching and an OpenAI-compatible server; the alternatives were
+evaluated and the reasoning is in [docs/engines/](docs/engines/).
 
-*Latest News* 🔥
+The work turned out to be less about kernels than about three things nobody
+profiles: how much memory is resident while the model *loads*, how much is
+resident that nothing ever reads, and whether the number you just measured came
+from the code path you thought it did. The largest single win here — 843 MB —
+was a setting that had been marked "no effect, do not retry" because the code
+path under test discarded it.
 
-- [2026/08] We released [0.28.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.28.0), featuring production-ready [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) serving on GPU and NPU, a unified AR/DiT paged KV cache runtime, and enhanced realtime full-duplex serving for the [MiniCPM-o series](recipes/OpenBMB/MiniCPM-o-4_5.md).
-- [2026/08] [VeRL-Omni](https://github.com/verl-project/verl-omni) `v0.2.0` is released: faster diffusion RL powered by vLLM-Omni (request-level/step-wise batching with FA3), rebuilt Qwen3-Omni multimodal training (DPO & GSPO), plus LTX-2.3, Qwen-Image-Edit support and more. See the [release notes](https://github.com/verl-project/verl-omni/releases/tag/v0.2.0).
-- [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support.
-- [2026/07] We released [0.24.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.24.0) - aligned with the vLLM 0.24 release line, expanding production-ready coverage across TTS, speech, diffusion, image/video generation, and robot-policy serving, with major Omni stage runtime refactoring, diffusion request-level batching, async output materialization, quantization/cache/memory improvements, and broad CUDA/ROCm/XPU/NPU support.
-- [2026/06] Starting with [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0), vLLM-Omni publishes a stable release aligned with every even-numbered upstream vLLM minor version. [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0), [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0), [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0), and [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) continued this cadence, expanding omni and world-model support with [NVIDIA Cosmos3](recipes/cosmos3/Cosmos3-Nano.md) and DreamZero, adding models such as MiniCPM-o 4.5, MOSS-TTS, and Lance, and advancing TTS, diffusion, distributed execution, quantization, RL integration through [VeRL-Omni](https://github.com/verl-project/verl-omni), and CUDA/ROCm/MUSA/NPU/XPU coverage.
-- [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
-- [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.
+This repository holds the engine changes, the measurement harness, the raw
+results, the baselines they are compared against, and the written analyses.
+Upstream's own README is preserved at [docs/README.vllm-omni.md](docs/README.vllm-omni.md).
 
----
+## What's here
 
-## About
+| area | where |
+|---|---|
+| 4-bit CPU quantization | `vllm_omni/model_executor/layers/quantization/cpu_int4.py` — W4A16 on PyTorch's tinygemm kernel, prefill reading the kernel's own weight layout instead of a duplicate |
+| Memory | RoPE tables sized to the servable context; weights repacked in blocks; a KV budget that is actually honoured on CPU |
+| CPU platform + vLLM patches | `vllm_omni/platforms/cpu/`, `vllm_omni/patch.py` — vLLM is monkey-patched at import; its *source* is unmodified and no vLLM is redistributed. See [docs/edge/environment.md](docs/edge/environment.md) |
+| Device routing | `vllm_omni/edge/weight_path.py` picks the weight format per device class, from measurements rather than assumption |
+| Device placement | `vllm_omni/edge/placement.py` decides which parameters leave the GPU when the checkpoint is bigger than the VRAM — cold ones first, because `embed_tokens` and `lm_head` are the same size and the opposite decision |
+| Measurement harness | `benchmarks/edge_harness/` — the grid runner, benchmarks, memory ledger, fidelity metric and agent suite every number here came from |
+| Raw results | `benchmarks/edge_harness/results/` — 100+ artifacts backing the published figures, each with provenance |
+| Baselines | `baselines/` — llama.cpp runs and the comparison scripts |
+| Engine analyses | `docs/engines/` — llama.cpp, ExecuTorch, MNN, ncnn, vLLM, vLLM-Omni |
+| Study documents | `docs/analysis/` — the full reports these summaries distil |
 
-[vLLM](https://github.com/vllm-project/vllm) was originally designed to support large language models for text-based autoregressive generation tasks. vLLM-Omni is a framework that extends its support for omni-modality model inference and serving:
+## Headline results
 
-- **Omni-modality**: Text, image, audio, video, and action data processing
-- **Non-autoregressive Architectures**: extend the AR support of vLLM to Diffusion Transformers (DiT) and other parallel generation models
-- **Heterogeneous outputs**: from traditional text generation to multimodal and action outputs
+Spark-X2.5-1.7B, Xeon Platinum 8480C, 24 pinned node-local cores, W4A16 g128.
+Every figure is a median of at least three interleaved passes against a control
+arm, on a shared host where end-to-end throughput swings ~30% run to run.
 
-<p align="center">
-  <picture>
-    <img alt="vllm-omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/omni-modality-model-architecture.png" width=55%>
-  </picture>
-</p>
+**Memory** — each measured as its own A/B with the alternative reachable by a flag:
 
-vLLM-Omni is fast with:
+| lever | saving | cost |
+|---|---|---|
+| KV budget 1 GiB → 192 MiB | **843 MB** | fewer cached tokens |
+| RoPE table capped to `max_model_len` | **618 MB** | none |
+| One packed weight layout instead of two | **615 MB** | ~1.6× prefill |
+| Blocked weight repack | **489 MB** of load-time peak | none |
 
-- State-of-the-art AR support by leveraging efficient KV cache management from vLLM
-- Pipelined stage execution overlapping for high throughput performance
-- Fully disaggregation based on OmniConnector and dynamic resource allocation across stages
+These overlap and **must not be added** — see [docs/edge/results.md](docs/edge/results.md).
 
-vLLM-Omni is flexible and easy to use with:
+**Speed**, same machine:
 
-- Heterogeneous pipeline abstraction to manage complex model workflows
-- Seamless integration with popular Hugging Face models
-- Tensor, pipeline, data and expert parallelism support for distributed inference
-- Streaming outputs
-- OpenAI-compatible API server
-- [Full-duplex realtime serving](docs/serving/realtime_duplex_api.md) with streaming audio input and output
+| path | decode | prefill (pp512) |
+|---|---|---|
+| W4A8 (AMX `int4_scaled_mm_cpu`) | 74.4 tok/s | — |
+| W4A16 (tinygemm) | 71.6 tok/s | 2190 tok/s |
+| `cpu_gemm_wna16` fallback | 32.3 tok/s | — |
+| llama.cpp Q4_K_M (reference) | 90.0 tok/s | 1121 tok/s |
 
-vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
+Decode is llama.cpp's; prefill is ours by ~2×. Neither engine is
+bandwidth-bound — both sit at 33–39% of the measured 222 GB/s ceiling — so the
+decode gap is the execution model, not the kernels. See
+[docs/edge/comparisons.md](docs/edge/comparisons.md).
 
-- **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
-- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
-- **Robot-policy and action models** (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
+## Why the numbers here are worth reading
 
-## Getting Started
+This project has retracted five conclusions, every one of them a measurement
+error rather than a coding error. The harness exists because of them:
 
-Visit our [documentation](https://vllm-omni.readthedocs.io/en/latest/) to learn more.
+- a "3.1 GB saving" that was `VmRSS` counting the mmap'd checkpoint;
+- W4A8 timings taken on a checkpoint whose output head was silently left in bf16;
+- a memory benchmark reading `/proc/self/status` while the model ran in a *worker process*;
+- "KV sizing saves only 188 MB" — measured through a code path that discarded the setting, so both arms ran the same cache;
+- a 1.9× prefill cost quoted from the one pass of a grid whose other passes had crashed.
 
-- [Installation](https://vllm-omni.readthedocs.io/en/latest/getting_started/installation/)
-- [Quickstart](https://vllm-omni.readthedocs.io/en/latest/getting_started/quickstart/)
-- [Nightly CUDA Docker images](https://hub.docker.com/r/vllm/vllm-omni/tags?name=nightly) built from the latest `main`
-- [List of Supported Models](https://vllm-omni.readthedocs.io/en/latest/models/supported_models/)
-- [Deployment Recipes](https://recipes.vllm.ai) for vLLM-Omni model serving
+[docs/edge/methodology.md](docs/edge/methodology.md) is the resulting discipline,
+and it is the most transferable thing in this repository.
 
-## Contributing
+## Quick start
 
-We welcome and value any contributions and collaborations.
-Please check out [Contributing to vLLM-Omni](https://vllm-omni.readthedocs.io/en/latest/contributing/) for how to get involved.
+```bash
+# CPU build of vLLM, then this fork
+pip install vllm==0.28.0+cpu
+VLLM_TARGET_DEVICE=cpu pip install -e .
 
-## Citation
-
-If you use vLLM-Omni for your research, please cite our [paper](https://arxiv.org/abs/2602.02204):
-
-```bibtex
-@article{yin2026vllmomni,
-  title={vLLM-Omni: Fully Disaggregated Serving for Any-to-Any Multimodal Models},
-  author={Peiqi Yin, Jiangyun Zhu, Han Gao, Chenguang Zheng, Yongxiang Huang, Taichang Zhou, Ruirui Yang, Weizhi Liu, Weiqing Chen, Canlin Guo, Didan Deng, Zifeng Mo, Cong Wang, James Cheng, Roger Wang, Hongsheng Liu},
-  journal={arXiv preprint arXiv:2602.02204},
-  year={2026}
-}
+# a right-sized edge deployment
+VLLM_CPU_KVCACHE_SPACE=1 python -c "
+from vllm import LLM, SamplingParams
+llm = LLM(model='<your-cpu_int4-checkpoint>', quantization='cpu_int4',
+          dtype='bfloat16', max_model_len=2048, max_num_seqs=1,
+          kv_cache_memory_bytes=192 << 20)   # the single largest memory lever
+print(llm.generate(['Hello'], SamplingParams(max_tokens=32))[0].outputs[0].text)
+"
 ```
 
-## Join the Community
+## Documentation
 
-Feel free to ask questions, provide feedbacks and discuss with fellow users of vLLM-Omni in `#sig-omni` slack channel at [slack.vllm.ai](https://slack.vllm.ai) or vLLM user forum at [discuss.vllm.ai](https://discuss.vllm.ai).
+| | |
+|---|---|
+| [docs/edge/results.md](docs/edge/results.md) | every measured figure, with spread and baseline |
+| [docs/edge/methodology.md](docs/edge/methodology.md) | how to measure on a shared machine without fooling yourself — the five retractions and the rules they left |
+| [docs/edge/cpu-4bit.md](docs/edge/cpu-4bit.md) | the two 4-bit paths, the packed layout as measured, building a checkpoint |
+| [docs/edge/comparisons.md](docs/edge/comparisons.md) | against llama.cpp and the other engines considered |
+| [docs/edge/environment.md](docs/edge/environment.md) | pins, machine, and why there is no vLLM patch series |
+| [docs/edge/local-text-mode.md](docs/edge/local-text-mode.md) | the single-stage local text engine: device capabilities, memory admission, explicit refusals, cancellation |
+| [docs/edge/external-stage-backends.md](docs/edge/external-stage-backends.md) | placing a stage on the Radeon 890M or the AMD NPU: out-of-process workers, proven placement, the shared-RAM ledger |
+| [docs/edge/harness.md](docs/edge/harness.md) | running a comparison |
+| [docs/edge/laptop-heterogeneous.md](docs/edge/laptop-heterogeneous.md) | Qwen3.8-27B and Spark-X2.5-4B on a Ryzen AI laptop — what the CPU, NPU and GPU are each worth when decode is bandwidth-bound |
+| [ROADMAP.md](ROADMAP.md) | what is next, with expected gains and how each would be verified — and what was rejected |
 
-## Star History
+## Licence
 
-<a href="https://www.star-history.com/?repos=vllm-project%2Fvllm-omni&type=date&legend=top-left">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=vllm-project/vllm-omni&type=date&theme=dark&legend=top-left&sealed_token=ExgLDZJoQEg27Zfhhut2LqN0GYO6Fw2PWLwPE6JYBUp2BgM3hmsYlwaIVopnUEfbRXidQ4nisumrTdKYydiKhy1SZXipw47qY2_tiUDhCpsPXeXtPuEVKVzBwKs3pw0tiHsJgtSfwXx5yjHXck0Y2SblzFWeJYCkTe1WLGTbUAOIETjXXQJjyCGZvKz5" />
-    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=vllm-project/vllm-omni&type=date&legend=top-left&sealed_token=ExgLDZJoQEg27Zfhhut2LqN0GYO6Fw2PWLwPE6JYBUp2BgM3hmsYlwaIVopnUEfbRXidQ4nisumrTdKYydiKhy1SZXipw47qY2_tiUDhCpsPXeXtPuEVKVzBwKs3pw0tiHsJgtSfwXx5yjHXck0Y2SblzFWeJYCkTe1WLGTbUAOIETjXXQJjyCGZvKz5" />
-    <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=vllm-project/vllm-omni&type=date&legend=top-left&sealed_token=ExgLDZJoQEg27Zfhhut2LqN0GYO6Fw2PWLwPE6JYBUp2BgM3hmsYlwaIVopnUEfbRXidQ4nisumrTdKYydiKhy1SZXipw47qY2_tiUDhCpsPXeXtPuEVKVzBwKs3pw0tiHsJgtSfwXx5yjHXck0Y2SblzFWeJYCkTe1WLGTbUAOIETjXXQJjyCGZvKz5" />
-  </picture>
-</a>
-
-## License
-
-Apache License 2.0, as found in the [LICENSE](./LICENSE) file.
+Apache-2.0, inherited from vLLM-Omni. See [LICENSE](LICENSE) and [NOTICE](NOTICE).

ROADMAP.md

@@ -0,0 +1,107 @@
+# Roadmap
+
+Ordered by whether the verification method is *correct*, not by expected size.
+That ordering is deliberate: this project retracted five conclusions, and in
+every case the fix was to the measurement rather than the code.
+
+Each item states the measurement it starts from, what it should gain, and how
+that gain would be verified. An item without a verification method is not on
+this list.
+
+## Done
+
+| | item | result |
+|---|---|---|
+| H1 | Task oracles judged by device state, with variants | alarm oracle needed a *saved* alarm, not a typed `7`; a Wi-Fi round trip now fails the already-satisfied task |
+| H2 | Harness semantics | arm order really rotates; `best` respects metric direction; provenance stamped; failures recorded |
+| H3 | The three missing measurements | `ready_s`, peak `RssAnon`, memory at a filled context — all previously unmeasurable |
+| H4 | Contention guard | aggregate stolen capacity + straggler rule, on the target cores rather than host load average |
+| H5 | Regression gates | bands from observed spread; throughput explicitly ungateable at ~30% swing |
+| H6 | Fidelity metric that ranks | top-1 agreement and KL, replacing a saturating 12-prompt greedy check |
+| H8 | Retracted numbers purged from source | two modules were still shipping withdrawn figures as documentation |
+| I0 | Blocked weight repack | **489 MB** of load-time peak, no cost |
+| I1 | One packed weight layout | **615 MB** resident, at ~1.63× prefill |
+| I2 | Measure `cpu_gemm_wna16` | 2.2× slower than tinygemm; routing was right, now for a measured reason |
+| I7 | Memory ledger, by attribution | found a **618 MB** RoPE table and an **843 MB** KV cache, both inside an "unexplained 1.6 GB" |
+
+## Next
+
+**I6 — import cost.** `import vllm_omni` is ~30 s against `import vllm`'s 5.2 s,
+with transformers' auto image-processing (4.6 s) and the `openai` client (1.3 s)
+identifiable inside it, for a text model that needs neither. Every stage
+subprocess pays it again.
+*Expect:* an upper bound of ~6 s per process; deferred imports often move cost
+rather than remove it. *Verify:* `ready_s`, which now exists — the old timer
+started after imports, so the benchmark meant to prove this could not see it.
+
+**Drop the row-major embedding copy.** `embed_tokens.weight_packed` is 128 MB,
+kept solely because the tinygemm layout was believed un-addressable for a
+single-row lookup. That premise is now known false. *Verify:* `RssAnon` plus an
+embedding-output SNR assertion.
+
+**I10 first step — why is the GEMM at 60%?** The decode GEMM sustains 132 GB/s
+against a measured 222 GB/s ceiling. Part is arithmetic (tinygemm dequantizes
+per activation row) and part may be that each of 113 calls streams only a few MB,
+too little for the prefetcher to reach steady state. Attribute before optimising.
+*Expect:* unknown — a diagnosis, whose output is whether the 2.76 ms implied by
+the gap is reachable at all.
+
+**I3 — asymmetric W4A8.** The symmetric grid is our exporter's choice, not the
+kernel's limit; `models/spark2_5-gptq-asym-g128` already exists. The cheapest
+untried fidelity experiment, and it tests the *stated* cause of W4A8's error.
+*Verify:* `fidelity_kl.py` and the scored agent suite.
+
+**I5 — attention wrapper consolidation.** Four ops per layer, 1.31 ms/step, of
+which 0.83 ms is wrapper self time; at batch 1 the cache-update loop runs two
+iterations while the wrapper rebuilds context and allocates a dummy tensor for
+compile ordering. *Expect:* 0.3–0.5 ms, revised down from an earlier 0.6–0.9 ms
+that assumed a workload never profiled. *Verify:* per-op profile **and**
+unprofiled step latency — one outer call that still launches two threaded
+kernels removes neither thread launch.
+
+**I4 — a better quantizer.** Ours reproduces 2/12 greedy prompts against its own
+bf16; Q4_K_M manages 4/12 — but at 4.5 bits against our 4.25, so close the
+budget gap first (g64, or fp16 scale+zero, both config changes) before
+implementing error-compensated rounding. Timebox it; a null result is worth
+recording.
+
+## Considered and rejected
+
+**A CPU megakernel.** On GPU this fuses a forward pass into one launch to remove
+launch overhead and the cross-kernel prefetch bubble. On CPU the analogues are
+OpenMP fork/join and ATen dispatch — and llama.cpp already is one:
+`ggml_threadpool` starts threads once and they walk the whole graph,
+synchronising at `ggml_barrier` through an atomic spin, where vLLM enters a
+separate `omp parallel` region per kernel.
+
+Measured before deciding: a parallel region costs ~1.2 µs at 24 threads, and
+spin-waiting does not help (`OMP_WAIT_POLICY=ACTIVE` moved it 1.23 → 1.18 µs), so
+it is barrier cost rather than thread wake-up. At a few hundred regions per
+decode step that is 0.3–0.4 ms, 2–3% — the same money as I5, far more cheaply.
+The larger prize is the streaming bubble, but capturing it means replacing the
+execution model inside vLLM's CPU backend, which reimplements ggml. Not next.
+
+**More kernel fusion.** Four attempts, four losses: vLLM's fused CPU RMSNorm is
+3.6–4.5× faster in isolation and slower in situ (inductor already fuses the
+native form with the residual add); fused `gelu_and_mul` 2.7× slower at Spark's
+shape; `cpp.min_chunk_size` −6% decode; an inline gate reduction −44%.
+
+**MiniCPM-o CPU overlay.** The 8 B thinker at bf16 is ~19 GB, so it buys no edge
+deployability. The argument for it — that three-stage parallel init should beat
+the two-stage 2.1× — is also not how overlap works: the bound is ΣT/max(T).
+
+**Speculative decoding for tg128.** It would inflate a fixed-prompt `ignore_eos`
+benchmark without representing anything. Measure it on the agent suite or not at
+all.
+
+**Thread count and warm-up batch size as memory levers.** 64 MB and 2 MB.
+
+## Harness work still open
+
+- **A real device in the agent loop.** The toolkit has 38 tests and has never
+  touched a phone — this host has no `adb`, no emulator, no `/dev/kvm`. The
+  real-device path and per-task resets exist; what is untested is whether a real
+  Settings app matches the fixtures.
+- **Success rates, not single runs.** The build comparison that produced
+  "navigation survives quantization, completion does not" is one run per cell.
+  Variants now exist so repeats are trials; they need running.

baselines/README.md

@@ -0,0 +1,32 @@
+# Baselines
+
+Measurements of other engines and of unquantized references, kept so the
+comparisons in [docs/edge/comparisons.md](../docs/edge/comparisons.md) can be
+checked rather than taken on trust.
+
+## `llama_cpp/`
+
+llama.cpp is the reference for CPU inference of a quantized model. Pinned
+commit and build flags are in
+[docs/edge/environment.md](../docs/edge/environment.md); no llama.cpp source is
+redistributed here, only output produced with it.
+
+| file | what |
+|---|---|
+| `llamacpp_q4km_t{4,8,16,32}.json` | Q4_K_M across thread counts |
+| `llamacpp_grid_t*.json`, `llamacpp_node1_t*.json` | thread and NUMA-node sweeps |
+| `ab_llamacpp_*.json` | interleaved A/B against vLLM on the same cores |
+| `il_llamacpp_p{1,2,3}.json` | the interleaved passes behind the headline decode figure |
+| `cmp_llamacpp.py` | the comparison driver |
+
+Both engines are pinned with `taskset` to the same physical cores and run
+strictly one at a time; vLLM's prefix caching is disabled, because `llama-bench`
+has no such cache and leaving it on would not be a comparison.
+
+## Quality references
+
+`quality_*.json` and `quality_cmp_*.json` hold the greedy-divergence comparison:
+each build against its **own** bf16 output over 12 prompts, plus the
+cross-engine roll-up. Read the caveats in
+[docs/edge/comparisons.md](../docs/edge/comparisons.md) before quoting these —
+the bit budgets are not matched, and the metric saturates.

baselines/llama_cpp/ab_llamacpp_after.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T18:59:20Z",
+    "avg_ns": 580079289,
+    "stddev_ns": 22445145,
+    "avg_ts": 883.695051,
+    "stddev_ts": 34.173372,
+    "samples_ns": [ 550140411, 579685176, 611186497, 588372185, 571012179 ],
+    "samples_ts": [ 930.671, 883.238, 837.715, 870.197, 896.653 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T18:59:24Z",
+    "avg_ns": 2118395683,
+    "stddev_ns": 31881177,
+    "avg_ts": 60.434035,
+    "stddev_ts": 0.909662,
+    "samples_ns": [ 2110731151, 2152277901, 2078510078, 2149464130, 2100995155 ],
+    "samples_ts": [ 60.6425, 59.4719, 61.5826, 59.5497, 60.9235 ]
+  }
+]

baselines/llama_cpp/ab_llamacpp_before.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T18:50:11Z",
+    "avg_ns": 453009837,
+    "stddev_ns": 4617262,
+    "avg_ts": 1130.311918,
+    "stddev_ts": 11.483962,
+    "samples_ns": [ 459248279, 451575469, 456242077, 449875010, 448108352 ],
+    "samples_ts": [ 1114.87, 1133.81, 1122.21, 1138.09, 1142.58 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T18:50:14Z",
+    "avg_ns": 1475847924,
+    "stddev_ns": 9833345,
+    "avg_ts": 86.732884,
+    "stddev_ts": 0.578067,
+    "samples_ns": [ 1489621433, 1475111972, 1476264136, 1476396077, 1461846002 ],
+    "samples_ts": [ 85.9279, 86.7731, 86.7054, 86.6976, 87.5605 ]
+  }
+]

baselines/llama_cpp/ab_llamacpp_t24.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T18:44:52Z",
+    "avg_ns": 487743656,
+    "stddev_ns": 25108865,
+    "avg_ts": 1051.969118,
+    "stddev_ts": 54.409083,
+    "samples_ns": [ 484545979, 492513415, 522973237, 486232711, 452452942 ],
+    "samples_ts": [ 1056.66, 1039.57, 979.018, 1052.99, 1131.61 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T18:44:55Z",
+    "avg_ns": 1589326576,
+    "stddev_ns": 47473222,
+    "avg_ts": 80.594928,
+    "stddev_ts": 2.413871,
+    "samples_ns": [ 1529730792, 1557930160, 1636596682, 1585818829, 1636556420 ],
+    "samples_ts": [ 83.6749, 82.1603, 78.2111, 80.7154, 78.213 ]
+  }
+]

baselines/llama_cpp/cmp_llamacpp.py

@@ -0,0 +1,106 @@
+"""Greedy-decode parity for llama.cpp against the same fp32 reference vLLM is
+scored on.
+
+The point of the comparison is that "4-bit" is not one thing: llama.cpp's
+Q4_K_M and our `cpu_int4` spend a similar number of bits per weight but pick
+different grids, so the speed comparison is only meaningful next to a quality
+comparison run on identical prompts.  llama-server takes a prompt as raw token
+ids, so both engines see exactly the token sequence the reference did.
+"""
+import argparse, json, subprocess, time, urllib.request
+
+import numpy as np
+
+
+def post(url, payload, timeout=600):
+    req = urllib.request.Request(
+        url, data=json.dumps(payload).encode(),
+        headers={"Content-Type": "application/json"})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return json.loads(r.read())
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--gguf", required=True)
+    ap.add_argument("--ref", default="ref.json")
+    ap.add_argument("--ref-logits", default="ref_logits.npy")
+    ap.add_argument("--bin", default="/data/zhoutaichang/embedding_infer/llama.cpp/build/bin/llama-server")
+    ap.add_argument("--cores", default="56-87")
+    ap.add_argument("--threads", type=int, default=32)
+    ap.add_argument("--port", type=int, default=18099)
+    ap.add_argument("--ctx", type=int, default=4096)
+    ap.add_argument("--out", default="parity_llamacpp.json")
+    a = ap.parse_args()
+
+    ref = json.load(open(a.ref))
+    ref_logits = np.load(a.ref_logits)
+    n_new = len(ref["new_ids"])
+
+    proc = subprocess.Popen(
+        ["taskset", "-c", a.cores, a.bin, "-m", a.gguf, "--port", str(a.port),
+         "-t", str(a.threads), "-c", str(a.ctx), "--no-warmup", "--host", "127.0.0.1"],
+        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    url = f"http://127.0.0.1:{a.port}"
+    try:
+        for _ in range(180):
+            try:
+                urllib.request.urlopen(url + "/health", timeout=2)
+                break
+            except Exception:
+                time.sleep(1)
+        else:
+            raise RuntimeError("llama-server did not become healthy")
+
+        res = post(url + "/completion", {
+            "prompt": ref["prompt_ids"], "n_predict": n_new,
+            "temperature": 0.0, "top_k": 1, "n_probs": 20,
+            "cache_prompt": False, "post_sampling_probs": False,
+        })
+    finally:
+        proc.terminate()
+        try:
+            proc.wait(timeout=30)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+
+    probs = res.get("completion_probabilities") or []
+    ids = [p["id"] for p in probs] if probs and "id" in probs[0] else []
+    n = min(n_new, len(ids)) if ids else 0
+    match = sum(int(x == y) for x, y in zip(ref["new_ids"][:n], ids[:n]))
+
+    rep = {
+        "engine": "llama.cpp", "gguf": a.gguf.split("/")[-1],
+        "ref_text": ref["text"], "llamacpp_text": res.get("content", ""),
+        "greedy_token_match": f"{match}/{n}" if n else "n/a",
+        "exact_match": bool(n) and ref["new_ids"][:n] == ids[:n],
+        "ref_top1": ref["top1"],
+        "llamacpp_first_token": ids[0] if ids else None,
+        "first_token_agree": bool(ids) and ids[0] == ref["top1"],
+    }
+    if probs:
+        import math
+        top = (probs[0].get("top_logprobs") or probs[0].get("top_probs")
+               or probs[0].get("probs") or [])
+        first = {}
+        for e in top:
+            if "id" not in e:
+                continue
+            first[int(e["id"])] = (float(e["prob"]) if "prob" in e
+                                   else math.exp(float(e["logprob"])))
+        ref_top = np.argsort(-ref_logits)[:20]
+        ref_p = np.exp(ref_logits - ref_logits.max())
+        ref_p /= ref_p.sum()
+        pairs = [(int(t), float(ref_p[t]), first[int(t)]) for t in ref_top if int(t) in first]
+        rep["top20_prob_overlap"] = f"{len(pairs)}/20"
+        if pairs:
+            d = [abs(r - v) for _, r, v in pairs]
+            rep["top20_prob_max_abs_diff"] = max(d)
+            rep["top20_prob_mean_abs_diff"] = sum(d) / len(d)
+    print(json.dumps(rep, indent=2, ensure_ascii=False))
+    with open(a.out, "w") as f:
+        json.dump(rep, f, indent=2, ensure_ascii=False)
+
+
+if __name__ == "__main__":
+    main()

baselines/llama_cpp/il_llamacpp_p1.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T19:04:49Z",
+    "avg_ns": 489021600,
+    "stddev_ns": 23921490,
+    "avg_ts": 1048.939735,
+    "stddev_ts": 49.910229,
+    "samples_ns": [ 527122937, 485801934, 462083348, 490951261, 479148522 ],
+    "samples_ts": [ 971.31, 1053.93, 1108.03, 1042.87, 1068.56 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T19:04:52Z",
+    "avg_ns": 1570942064,
+    "stddev_ns": 20220055,
+    "avg_ts": 81.490699,
+    "stddev_ts": 1.061319,
+    "samples_ns": [ 1586700558, 1582528379, 1537750061, 1565679809, 1582051516 ],
+    "samples_ts": [ 80.6705, 80.8832, 83.2385, 81.7536, 80.9076 ]
+  }
+]

baselines/llama_cpp/il_llamacpp_p2.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T19:14:20Z",
+    "avg_ns": 464927159,
+    "stddev_ns": 22146638,
+    "avg_ts": 1103.200971,
+    "stddev_ts": 51.267874,
+    "samples_ns": [ 453191976, 479254490, 448568607, 496875740, 446744984 ],
+    "samples_ts": [ 1129.76, 1068.33, 1141.41, 1030.44, 1146.07 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T19:14:23Z",
+    "avg_ns": 1422234679,
+    "stddev_ns": 12240631,
+    "avg_ts": 90.004494,
+    "stddev_ts": 0.767183,
+    "samples_ns": [ 1422285965, 1416723437, 1414194914, 1414611159, 1443357923 ],
+    "samples_ts": [ 89.996, 90.3493, 90.5109, 90.4842, 88.6821 ]
+  }
+]

baselines/llama_cpp/il_llamacpp_p3.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T19:20:46Z",
+    "avg_ns": 456731353,
+    "stddev_ns": 6199988,
+    "avg_ts": 1121.174997,
+    "stddev_ts": 15.277320,
+    "samples_ns": [ 459406023, 462620142, 461507407, 450329050, 449794146 ],
+    "samples_ts": [ 1114.48, 1106.74, 1109.41, 1136.95, 1138.3 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T19:20:49Z",
+    "avg_ns": 1506056809,
+    "stddev_ns": 24193761,
+    "avg_ts": 85.007660,
+    "stddev_ts": 1.362252,
+    "samples_ns": [ 1478402259, 1532453276, 1530685126, 1494657866, 1494085519 ],
+    "samples_ts": [ 86.58, 83.5262, 83.6227, 85.6383, 85.6711 ]
+  }
+]

baselines/llama_cpp/llamacpp_grid_t16.json

@@ -0,0 +1,272 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:32:06Z",
+    "avg_ns": 1498583726,
+    "stddev_ns": 7171856,
+    "avg_ts": 341.662152,
+    "stddev_ts": 1.628172,
+    "samples_ns": [ 1493166066, 1510351399, 1500283975, 1493415633, 1495701559 ],
+    "samples_ts": [ 342.896, 338.994, 341.269, 342.838, 342.314 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:32:18Z",
+    "avg_ns": 5062657211,
+    "stddev_ns": 186447127,
+    "avg_ts": 25.309483,
+    "stddev_ts": 0.893384,
+    "samples_ns": [ 4950579569, 4934896396, 5035243526, 5388250087, 5004316480 ],
+    "samples_ts": [ 25.8556, 25.9377, 25.4208, 23.7554, 25.5779 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:32:43Z",
+    "avg_ns": 560100709,
+    "stddev_ns": 15689530,
+    "avg_ts": 914.690155,
+    "stddev_ts": 25.391149,
+    "samples_ns": [ 566521408, 541347341, 583132866, 553334911, 556167020 ],
+    "samples_ts": [ 903.761, 945.788, 878.016, 925.299, 920.587 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:32:47Z",
+    "avg_ns": 1933840318,
+    "stddev_ns": 51036422,
+    "avg_ts": 66.225568,
+    "stddev_ts": 1.706730,
+    "samples_ns": [ 1896818220, 1923412956, 1937369806, 1892702916, 2018897692 ],
+    "samples_ts": [ 67.4814, 66.5484, 66.069, 67.6282, 63.4009 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:32:57Z",
+    "avg_ns": 635046952,
+    "stddev_ns": 3342779,
+    "avg_ts": 806.257449,
+    "stddev_ts": 4.239145,
+    "samples_ns": [ 634661823, 637237702, 639487443, 631413242, 632434552 ],
+    "samples_ts": [ 806.729, 803.468, 800.641, 810.879, 809.57 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:33:01Z",
+    "avg_ns": 1790378809,
+    "stddev_ns": 14069215,
+    "avg_ts": 71.496788,
+    "stddev_ts": 0.562779,
+    "samples_ns": [ 1803158642, 1794510633, 1803271959, 1776172087, 1774780727 ],
+    "samples_ts": [ 70.9865, 71.3286, 70.9821, 72.0651, 72.1216 ]
+  }
+]

baselines/llama_cpp/llamacpp_grid_t32.json

@@ -0,0 +1,272 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:35:02Z",
+    "avg_ns": 1017103540,
+    "stddev_ns": 116210707,
+    "avg_ts": 509.454206,
+    "stddev_ts": 66.492143,
+    "samples_ns": [ 1045283058, 1099078888, 1103686903, 1017870763, 819598088 ],
+    "samples_ts": [ 489.819, 465.845, 463.9, 503.011, 624.696 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:35:08Z",
+    "avg_ns": 4481043455,
+    "stddev_ns": 53485519,
+    "avg_ts": 28.568042,
+    "stddev_ts": 0.342133,
+    "samples_ns": [ 4526208574, 4537101214, 4480636429, 4454908368, 4406362692 ],
+    "samples_ts": [ 28.2797, 28.2118, 28.5674, 28.7324, 29.0489 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:35:31Z",
+    "avg_ns": 343075990,
+    "stddev_ns": 8202776,
+    "avg_ts": 1493.062250,
+    "stddev_ts": 35.634573,
+    "samples_ns": [ 333407890, 348737139, 337165270, 342580383, 353489272 ],
+    "samples_ts": [ 1535.66, 1468.15, 1518.54, 1494.54, 1448.42 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:35:33Z",
+    "avg_ns": 1673245422,
+    "stddev_ns": 44193699,
+    "avg_ts": 76.540484,
+    "stddev_ts": 2.008808,
+    "samples_ns": [ 1727982320, 1650954337, 1650417455, 1711748955, 1625124045 ],
+    "samples_ts": [ 74.0748, 77.5309, 77.5561, 74.7773, 78.7632 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:35:42Z",
+    "avg_ns": 433558704,
+    "stddev_ns": 61442386,
+    "avg_ts": 1198.988860,
+    "stddev_ts": 159.663627,
+    "samples_ns": [ 384355143, 403569845, 520552799, 475384982, 383930753 ],
+    "samples_ts": [ 1332.1, 1268.68, 983.57, 1077.02, 1333.57 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:35:44Z",
+    "avg_ns": 1574917625,
+    "stddev_ns": 239068571,
+    "avg_ts": 82.814413,
+    "stddev_ts": 12.730560,
+    "samples_ns": [ 1383407468, 1295986814, 1572559050, 1776956669, 1845678125 ],
+    "samples_ts": [ 92.5252, 98.7664, 81.396, 72.0333, 69.3512 ]
+  }
+]

baselines/llama_cpp/llamacpp_grid_t4.json

@@ -0,0 +1,272 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:19:27Z",
+    "avg_ns": 7115257366,
+    "stddev_ns": 607044890,
+    "avg_ts": 72.394533,
+    "stddev_ts": 6.405320,
+    "samples_ns": [ 7765939766, 6729867579, 7316853262, 7495192180, 6268434044 ],
+    "samples_ts": [ 65.9289, 76.0788, 69.9754, 68.3105, 81.6791 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:20:09Z",
+    "avg_ns": 30700265663,
+    "stddev_ns": 4064662246,
+    "avg_ts": 4.196201,
+    "stddev_ts": 0.381497,
+    "samples_ns": [ 32683898432, 32626033248, 27736648629, 27745563194, 32709184813 ],
+    "samples_ts": [ 3.9163, 3.92325, 4.61483, 4.61335, 3.91327 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:22:44Z",
+    "avg_ns": 4412049470,
+    "stddev_ns": 642897239,
+    "avg_ts": 118.057116,
+    "stddev_ts": 17.378814,
+    "samples_ns": [ 4619378132, 3608885475, 4557507327, 3988149647, 5286326771 ],
+    "samples_ts": [ 110.837, 141.872, 112.342, 128.38, 96.8536 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:23:12Z",
+    "avg_ns": 13160648883,
+    "stddev_ns": 4037164376,
+    "avg_ts": 9.845522,
+    "stddev_ts": 1.260348,
+    "samples_ns": [ 12727171856, 13397387109, 13869421316, 15023258698, 10786005440 ],
+    "samples_ts": [ 10.0572, 9.5541, 9.22894, 8.52012, 11.8672 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:24:19Z",
+    "avg_ns": 4955359522,
+    "stddev_ns": 278495888,
+    "avg_ts": 103.589056,
+    "stddev_ts": 5.933882,
+    "samples_ns": [ 4969396818, 4806604145, 5198757330, 4567278377, 5234760941 ],
+    "samples_ts": [ 103.031, 106.52, 98.4851, 112.102, 97.8077 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:24:49Z",
+    "avg_ns": 12648866627,
+    "stddev_ns": 4253557830,
+    "avg_ts": 10.355908,
+    "stddev_ts": 1.831931,
+    "samples_ns": [ 12076753955, 12558032815, 15171316078, 9645037747, 13793192540 ],
+    "samples_ts": [ 10.5989, 10.1927, 8.43697, 13.2711, 9.27994 ]
+  }
+]

baselines/llama_cpp/llamacpp_grid_t8.json

@@ -0,0 +1,272 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:28:17Z",
+    "avg_ns": 2946090115,
+    "stddev_ns": 19481720,
+    "avg_ts": 173.795716,
+    "stddev_ts": 1.144798,
+    "samples_ns": [ 2976783425, 2943842928, 2936842142, 2948762947, 2924219135 ],
+    "samples_ts": [ 171.998, 173.922, 174.337, 173.632, 175.089 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:28:34Z",
+    "avg_ns": 7926507628,
+    "stddev_ns": 166210109,
+    "avg_ts": 16.153997,
+    "stddev_ts": 0.336782,
+    "samples_ns": [ 7746359669, 7836100448, 8141583693, 8059318007, 7849176324 ],
+    "samples_ts": [ 16.5239, 16.3347, 15.7218, 15.8822, 16.3074 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:29:15Z",
+    "avg_ns": 1044627273,
+    "stddev_ns": 8668625,
+    "avg_ts": 490.153690,
+    "stddev_ts": 4.025039,
+    "samples_ns": [ 1040367711, 1059967868, 1042470115, 1039018745, 1041311927 ],
+    "samples_ts": [ 492.134, 483.034, 491.141, 492.773, 491.687 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:29:21Z",
+    "avg_ns": 2844703994,
+    "stddev_ns": 50284844,
+    "avg_ts": 45.007005,
+    "stddev_ts": 0.785784,
+    "samples_ns": [ 2841360663, 2852415251, 2800656934, 2925026017, 2804061106 ],
+    "samples_ts": [ 45.0488, 44.8743, 45.7036, 43.7603, 45.6481 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:29:36Z",
+    "avg_ns": 1179360692,
+    "stddev_ns": 9282489,
+    "avg_ts": 434.155009,
+    "stddev_ts": 3.414034,
+    "samples_ns": [ 1190805907, 1173513176, 1168400903, 1186783926, 1177299549 ],
+    "samples_ts": [ 429.961, 436.297, 438.206, 431.418, 434.894 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:29:43Z",
+    "avg_ns": 3018952734,
+    "stddev_ns": 27599225,
+    "avg_ts": 42.401651,
+    "stddev_ts": 0.388562,
+    "samples_ns": [ 3042267389, 2995063052, 3045385324, 2984815648, 3027232258 ],
+    "samples_ts": [ 42.0739, 42.737, 42.0308, 42.8837, 42.2828 ]
+  }
+]

baselines/llama_cpp/llamacpp_node0_warm_t32.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:47:23Z",
+    "avg_ns": 377579804,
+    "stddev_ns": 6351193,
+    "avg_ts": 1356.308108,
+    "stddev_ts": 22.545912,
+    "samples_ns": [ 372424973, 373894780, 373636992, 387431983, 380510292 ],
+    "samples_ts": [ 1374.77, 1369.37, 1370.31, 1321.52, 1345.56 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:47:25Z",
+    "avg_ns": 1305937293,
+    "stddev_ns": 13522740,
+    "avg_ts": 98.022226,
+    "stddev_ts": 1.005768,
+    "samples_ns": [ 1300442432, 1292366252, 1304438971, 1303902694, 1328536116 ],
+    "samples_ts": [ 98.428, 99.0431, 98.1265, 98.1668, 96.3466 ]
+  }
+]

baselines/llama_cpp/llamacpp_node1_t16.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:19:16Z",
+    "avg_ns": 640698554,
+    "stddev_ns": 8442449,
+    "avg_ts": 799.237980,
+    "stddev_ts": 10.456116,
+    "samples_ns": [ 652875468, 635459867, 646229581, 635186003, 633741853 ],
+    "samples_ts": [ 784.223, 805.716, 792.288, 806.063, 807.9 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:19:20Z",
+    "avg_ns": 1748941710,
+    "stddev_ns": 13313898,
+    "avg_ts": 73.190522,
+    "stddev_ts": 0.559300,
+    "samples_ns": [ 1759700788, 1762032558, 1728868153, 1744622542, 1749484513 ],
+    "samples_ts": [ 72.7396, 72.6434, 74.0369, 73.3683, 73.1644 ]
+  }
+]

baselines/llama_cpp/llamacpp_node1_t32.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:21:27Z",
+    "avg_ns": 379833208,
+    "stddev_ns": 2916251,
+    "avg_ts": 1348.023365,
+    "stddev_ts": 10.304461,
+    "samples_ns": [ 377554426, 384109520, 377555613, 381602693, 378343790 ],
+    "samples_ts": [ 1356.1, 1332.95, 1356.09, 1341.71, 1353.27 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:21:29Z",
+    "avg_ns": 1285969483,
+    "stddev_ns": 16587912,
+    "avg_ts": 99.548971,
+    "stddev_ts": 1.276520,
+    "samples_ns": [ 1279748126, 1273538113, 1309616758, 1270516087, 1296428332 ],
+    "samples_ts": [ 100.02, 100.507, 97.7385, 100.746, 98.7328 ]
+  }
+]

baselines/llama_cpp/llamacpp_node1_t4.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:14:01Z",
+    "avg_ns": 2437249430,
+    "stddev_ns": 66433948,
+    "avg_ts": 210.202021,
+    "stddev_ts": 5.922750,
+    "samples_ns": [ 2468232968, 2479407287, 2475612647, 2441614629, 2321379621 ],
+    "samples_ts": [ 207.436, 206.501, 206.817, 209.697, 220.558 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:14:15Z",
+    "avg_ns": 5289126007,
+    "stddev_ns": 63024171,
+    "avg_ts": 24.203349,
+    "stddev_ts": 0.288883,
+    "samples_ns": [ 5363916055, 5306624609, 5325596808, 5238325957, 5211166606 ],
+    "samples_ts": [ 23.8632, 24.1208, 24.0349, 24.4353, 24.5626 ]
+  }
+]

baselines/llama_cpp/llamacpp_node1_t8.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:16:53Z",
+    "avg_ns": 1253292754,
+    "stddev_ns": 79129486,
+    "avg_ts": 409.796891,
+    "stddev_ts": 25.209212,
+    "samples_ns": [ 1190927743, 1190669420, 1208414068, 1318310015, 1358142528 ],
+    "samples_ts": [ 429.917, 430.01, 423.696, 388.376, 376.985 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:17:01Z",
+    "avg_ns": 2909185423,
+    "stddev_ns": 35614966,
+    "avg_ts": 44.003861,
+    "stddev_ts": 0.540189,
+    "samples_ns": [ 2935902159, 2946667991, 2878606593, 2919344964, 2865405409 ],
+    "samples_ts": [ 43.5982, 43.4389, 44.466, 43.8455, 44.6708 ]
+  }
+]

baselines/llama_cpp/llamacpp_q4km_t16.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:19:51Z",
+    "avg_ns": 650543418,
+    "stddev_ns": 18555552,
+    "avg_ts": 787.546056,
+    "stddev_ts": 22.428007,
+    "samples_ns": [ 632032527, 670684809, 649115565, 632738336, 668145853 ],
+    "samples_ts": [ 810.085, 763.399, 788.766, 809.181, 766.3 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:19:55Z",
+    "avg_ns": 1781362880,
+    "stddev_ns": 21643541,
+    "avg_ts": 71.863545,
+    "stddev_ts": 0.869304,
+    "samples_ns": [ 1806353622, 1764741597, 1764660660, 1767373240, 1803685282 ],
+    "samples_ts": [ 70.861, 72.5319, 72.5352, 72.4239, 70.9658 ]
+  }
+]

baselines/llama_cpp/llamacpp_q4km_t32.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:21:20Z",
+    "avg_ns": 377608322,
+    "stddev_ns": 6410592,
+    "avg_ts": 1356.211504,
+    "stddev_ts": 22.766437,
+    "samples_ns": [ 372722694, 373942297, 373069875, 387378103, 380928645 ],
+    "samples_ts": [ 1373.68, 1369.2, 1372.4, 1321.71, 1344.08 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:21:22Z",
+    "avg_ns": 1342539204,
+    "stddev_ns": 21735221,
+    "avg_ts": 95.361608,
+    "stddev_ts": 1.535356,
+    "samples_ns": [ 1368242693, 1364047511, 1323244630, 1330069532, 1327091655 ],
+    "samples_ts": [ 93.5507, 93.8384, 96.7319, 96.2356, 96.4515 ]
+  }
+]

baselines/llama_cpp/llamacpp_q4km_t4.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:14:44Z",
+    "avg_ns": 2304834712,
+    "stddev_ns": 9004426,
+    "avg_ts": 222.144447,
+    "stddev_ts": 0.865236,
+    "samples_ns": [ 2319430963, 2305311298, 2304610521, 2298020765, 2296800014 ],
+    "samples_ts": [ 220.744, 222.096, 222.163, 222.8, 222.919 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:14:58Z",
+    "avg_ns": 5335792102,
+    "stddev_ns": 59840580,
+    "avg_ts": 23.991343,
+    "stddev_ts": 0.267723,
+    "samples_ns": [ 5425666503, 5348961684, 5341956156, 5275207674, 5287168494 ],
+    "samples_ts": [ 23.5916, 23.9299, 23.9613, 24.2644, 24.2096 ]
+  }
+]

baselines/llama_cpp/llamacpp_q4km_t8.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:17:31Z",
+    "avg_ns": 1911672545,
+    "stddev_ns": 4048624965,
+    "avg_ts": 363.751611,
+    "stddev_ts": 143.367368,
+    "samples_ns": [ 4771767171, 1201291347, 1195946935, 1197665158, 1191692115 ],
+    "samples_ts": [ 107.298, 426.208, 428.113, 427.498, 429.641 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:17:46Z",
+    "avg_ns": 2882701640,
+    "stddev_ns": 15288789,
+    "avg_ts": 44.403796,
+    "stddev_ts": 0.236813,
+    "samples_ns": [ 2894272938, 2890318648, 2890951071, 2881201632, 2856763912 ],
+    "samples_ts": [ 44.2253, 44.2858, 44.2761, 44.4259, 44.8059 ]
+  }
+]

baselines/quality_cmp_llamacpp-Q4_K_M.json

@@ -0,0 +1,80 @@
+{
+  "label": "llamacpp-Q4_K_M",
+  "prompts": 12,
+  "exact_match": "4/12",
+  "mean_prefix_kept": 0.5101,
+  "detail": [
+    {
+      "prompt": "chat_short",
+      "exact": false,
+      "common_prefix_chars": 91,
+      "ref_chars": 182
+    },
+    {
+      "prompt": "chat_long",
+      "exact": false,
+      "common_prefix_chars": 43,
+      "ref_chars": 246
+    },
+    {
+      "prompt": "chat_tools",
+      "exact": false,
+      "common_prefix_chars": 15,
+      "ref_chars": 290
+    },
+    {
+      "prompt": "plain_0",
+      "exact": true,
+      "common_prefix_chars": 361,
+      "ref_chars": 361
+    },
+    {
+      "prompt": "plain_1",
+      "exact": true,
+      "common_prefix_chars": 512,
+      "ref_chars": 512
+    },
+    {
+      "prompt": "plain_2",
+      "exact": false,
+      "common_prefix_chars": 71,
+      "ref_chars": 172
+    },
+    {
+      "prompt": "plain_3",
+      "exact": false,
+      "common_prefix_chars": 3,
+      "ref_chars": 352
+    },
+    {
+      "prompt": "plain_4",
+      "exact": true,
+      "common_prefix_chars": 318,
+      "ref_chars": 318
+    },
+    {
+      "prompt": "plain_5",
+      "exact": true,
+      "common_prefix_chars": 346,
+      "ref_chars": 346
+    },
+    {
+      "prompt": "plain_6",
+      "exact": false,
+      "common_prefix_chars": 2,
+      "ref_chars": 177
+    },
+    {
+      "prompt": "plain_7",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 320
+    },
+    {
+      "prompt": "plain_8",
+      "exact": false,
+      "common_prefix_chars": 2,
+      "ref_chars": 184
+    }
+  ]
+}

baselines/quality_cmp_vllm-W4A16-tinygemm.json

@@ -0,0 +1,80 @@
+{
+  "label": "vllm-W4A16-tinygemm",
+  "prompts": 12,
+  "exact_match": "2/12",
+  "mean_prefix_kept": 0.301,
+  "detail": [
+    {
+      "prompt": "chat_short",
+      "exact": false,
+      "common_prefix_chars": 91,
+      "ref_chars": 182
+    },
+    {
+      "prompt": "chat_long",
+      "exact": false,
+      "common_prefix_chars": 0,
+      "ref_chars": 261
+    },
+    {
+      "prompt": "chat_tools",
+      "exact": false,
+      "common_prefix_chars": 56,
+      "ref_chars": 287
+    },
+    {
+      "prompt": "plain_0",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 361
+    },
+    {
+      "prompt": "plain_1",
+      "exact": true,
+      "common_prefix_chars": 512,
+      "ref_chars": 512
+    },
+    {
+      "prompt": "plain_2",
+      "exact": false,
+      "common_prefix_chars": 4,
+      "ref_chars": 172
+    },
+    {
+      "prompt": "plain_3",
+      "exact": false,
+      "common_prefix_chars": 34,
+      "ref_chars": 352
+    },
+    {
+      "prompt": "plain_4",
+      "exact": true,
+      "common_prefix_chars": 318,
+      "ref_chars": 318
+    },
+    {
+      "prompt": "plain_5",
+      "exact": false,
+      "common_prefix_chars": 5,
+      "ref_chars": 346
+    },
+    {
+      "prompt": "plain_6",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 283
+    },
+    {
+      "prompt": "plain_7",
+      "exact": false,
+      "common_prefix_chars": 54,
+      "ref_chars": 320
+    },
+    {
+      "prompt": "plain_8",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 184
+    }
+  ]
+}

baselines/quality_cmp_vllm-W4A16.json

@@ -0,0 +1,80 @@
+{
+  "label": "vllm-W4A16",
+  "prompts": 12,
+  "exact_match": "2/12",
+  "mean_prefix_kept": 0.301,
+  "detail": [
+    {
+      "prompt": "chat_short",
+      "exact": false,
+      "common_prefix_chars": 91,
+      "ref_chars": 182
+    },
+    {
+      "prompt": "chat_long",
+      "exact": false,
+      "common_prefix_chars": 0,
+      "ref_chars": 261
+    },
+    {
+      "prompt": "chat_tools",
+      "exact": false,
+      "common_prefix_chars": 56,
+      "ref_chars": 287
+    },
+    {
+      "prompt": "plain_0",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 361
+    },
+    {
+      "prompt": "plain_1",
+      "exact": true,
+      "common_prefix_chars": 512,
+      "ref_chars": 512
+    },
+    {
+      "prompt": "plain_2",
+      "exact": false,
+      "common_prefix_chars": 4,
+      "ref_chars": 172
+    },
+    {
+      "prompt": "plain_3",
+      "exact": false,
+      "common_prefix_chars": 34,
+      "ref_chars": 352
+    },
+    {
+      "prompt": "plain_4",
+      "exact": true,
+      "common_prefix_chars": 318,
+      "ref_chars": 318
+    },
+    {
+      "prompt": "plain_5",
+      "exact": false,
+      "common_prefix_chars": 5,
+      "ref_chars": 346
+    },
+    {
+      "prompt": "plain_6",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 283
+    },
+    {
+      "prompt": "plain_7",
+      "exact": false,
+      "common_prefix_chars": 54,
+      "ref_chars": 320
+    },
+    {
+      "prompt": "plain_8",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 184
+    }
+  ]
+}

baselines/quality_cmp_vllm-W4A8-gptq.json

@@ -0,0 +1,80 @@
+{
+  "label": "vllm-W4A8-gptq",
+  "prompts": 12,
+  "exact_match": "1/12",
+  "mean_prefix_kept": 0.1263,
+  "detail": [
+    {
+      "prompt": "chat_short",
+      "exact": false,
+      "common_prefix_chars": 66,
+      "ref_chars": 182
+    },
+    {
+      "prompt": "chat_long",
+      "exact": false,
+      "common_prefix_chars": 14,
+      "ref_chars": 261
+    },
+    {
+      "prompt": "chat_tools",
+      "exact": false,
+      "common_prefix_chars": 16,
+      "ref_chars": 287
+    },
+    {
+      "prompt": "plain_0",
+      "exact": false,
+      "common_prefix_chars": 25,
+      "ref_chars": 361
+    },
+    {
+      "prompt": "plain_1",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 512
+    },
+    {
+      "prompt": "plain_2",
+      "exact": false,
+      "common_prefix_chars": 4,
+      "ref_chars": 172
+    },
+    {
+      "prompt": "plain_3",
+      "exact": false,
+      "common_prefix_chars": 4,
+      "ref_chars": 352
+    },
+    {
+      "prompt": "plain_4",
+      "exact": true,
+      "common_prefix_chars": 318,
+      "ref_chars": 318
+    },
+    {
+      "prompt": "plain_5",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 346
+    },
+    {
+      "prompt": "plain_6",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 283
+    },
+    {
+      "prompt": "plain_7",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 320
+    },
+    {
+      "prompt": "plain_8",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 184
+    }
+  ]
+}

baselines/quality_cmp_vllm-W4A8.json

@@ -0,0 +1,80 @@
+{
+  "label": "vllm-W4A8",
+  "prompts": 12,
+  "exact_match": "1/12",
+  "mean_prefix_kept": 0.1263,
+  "detail": [
+    {
+      "prompt": "chat_short",
+      "exact": false,
+      "common_prefix_chars": 66,
+      "ref_chars": 182
+    },
+    {
+      "prompt": "chat_long",
+      "exact": false,
+      "common_prefix_chars": 14,
+      "ref_chars": 261
+    },
+    {
+      "prompt": "chat_tools",
+      "exact": false,
+      "common_prefix_chars": 16,
+      "ref_chars": 287
+    },
+    {
+      "prompt": "plain_0",
+      "exact": false,
+      "common_prefix_chars": 25,
+      "ref_chars": 361
+    },
+    {
+      "prompt": "plain_1",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 512
+    },
+    {
+      "prompt": "plain_2",
+      "exact": false,
+      "common_prefix_chars": 4,
+      "ref_chars": 172
+    },
+    {
+      "prompt": "plain_3",
+      "exact": false,
+      "common_prefix_chars": 4,
+      "ref_chars": 352
+    },
+    {
+      "prompt": "plain_4",
+      "exact": true,
+      "common_prefix_chars": 318,
+      "ref_chars": 318
+    },
+    {
+      "prompt": "plain_5",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 346
+    },
+    {
+      "prompt": "plain_6",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 283
+    },
+    {
+      "prompt": "plain_7",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 320
+    },
+    {
+      "prompt": "plain_8",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 184
+    }
+  ]
+}

baselines/quality_llamacpp_bf16.json

@@ -0,0 +1,50 @@
+{
+  "chat_short": {
+    "text": "We are asked: \"What is the capital of France? Answer in one word.\" So the answer should be \"Paris\". But we need to output it in one word. So the final answer is \"Paris\".</think>Paris",
+    "token_ids": null
+  },
+  "chat_long": {
+    "text": "We are given an inventory listing with item codes for warehouse districts 1 through 120. The question asks: \"What is the code for warehouse district 73?\" We need to find the code corresponding to district 73.\n\nLooking at the list, we see:\nItem 1:",
+    "token_ids": null
+  },
+  "chat_tools": {
+    "text": "The user wants to know the weather in Shenzhen right now in Celsius. I have a function available to get weather for a city. Let me call it with the city \"Shenzhen\" and unit \"celsius\".</think><tool_call>get_weather<arg_key>city</arg_key><arg_value>Shenzhen</arg_value><arg_key>unit</arg_key>",
+    "token_ids": null
+  },
+  "plain_0": {
+    "text": " the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia",
+    "token_ids": null
+  },
+  "plain_1": {
+    "text": " mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission",
+    "token_ids": null
+  },
+  "plain_2": {
+    "text": "    a, b = 0, 1\n    for _ in range(n):\n        a, b = b, a + b\n        if _ == n:\n            return a\n    return a\n\n\ndef main():\n    n = int(input())\n    print(fibonacci(n",
+    "token_ids": null
+  },
+  "plain_3": {
+    "text": " the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that",
+    "token_ids": null
+  },
+  "plain_4": {
+    "text": " to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to",
+    "token_ids": null
+  },
+  "plain_5": {
+    "text": " the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the",
+    "token_ids": null
+  },
+  "plain_6": {
+    "text": " carbon and oxygen into glucose and other organic compounds.\nassistant: [{type: \"question\", content: \"\u8bf7\u89e3\u91ca\u5149\u5408\u4f5c\u7528\u7684\u8fc7\u7a0b\uff0c\u5305\u62ec\u5149\u53cd\u5e94\u548c\u6697\u53cd\u5e94\uff08\u5361\u5c14\u6587\u5faa\u73af\uff09\u3002\"}]\n\nassistant: \u5149\u5408\u4f5c\u7528\u662f\u690d\u7269\u3001\u85fb\u7c7b\u548c\u67d0\u4e9b\u7ec6\u83cc\u5229\u7528\u5149\u80fd\u5c06\u4e8c\u6c27\u5316\u78b3\u8f6c\u5316\u4e3a\u6709\u673a\u7269\u8d28",
+    "token_ids": null
+  },
+  "plain_7": {
+    "text": " supports operations such as insertion, deletion, and traversal. A binary search tree is a tree where each node has at most two children, and the left and right children are such that the left subtree contains all nodes with values less than the node's value, and the right subtree contains all nodes with values greater",
+    "token_ids": null
+  },
+  "plain_8": {
+    "text": " 1333. The Treaty of Westphalia was signed in 1333. The Treaty of Westphalia was signed in 1333. The Treaty of Westphalia was signed in 1333. The Treaty of Westphalia was signed in 133",
+    "token_ids": null
+  }
+}

baselines/quality_llamacpp_q4km.json

@@ -0,0 +1,50 @@
+{
+  "chat_short": {
+    "text": "We are asked: \"What is the capital of France? Answer in one word.\" So the answer should be a single word: \"Paris\". But we need to output just the answer in one word. The instruction says: \"Answer in one word.\" So we should respond with \"Paris\".</think>Paris",
+    "token_ids": null
+  },
+  "chat_long": {
+    "text": "We are given an inventory listing with items 1 through 120. Each item has a code for a warehouse district. The codes are given as:\n\nItem 1: 1007\nItem 2: 1014\nItem 3: 1021\n",
+    "token_ids": null
+  },
+  "chat_tools": {
+    "text": "The user wants weather for Shenzhen in celsius. I have the weather function available. Let me call it with the city \"Shenzhen\" and unit \"celsius\".</think><tool_call>get_weather<arg_key>city</arg_key><arg_value>Shenzhen</arg_value><arg_key>unit</arg_key><arg_value>celsius</arg_value></tool_call>",
+    "token_ids": null
+  },
+  "plain_0": {
+    "text": " the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia",
+    "token_ids": null
+  },
+  "plain_1": {
+    "text": " mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission",
+    "token_ids": null
+  },
+  "plain_2": {
+    "text": "    a, b = 0, 1\n    for _ in range(n):\n        a, b = b, a + b\n        return b\n\n\nfib = fibonacci\n\n\nprint(fib(10))\nprint(fib(20))\nprint(fib",
+    "token_ids": null
+  },
+  "plain_3": {
+    "text": " that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that",
+    "token_ids": null
+  },
+  "plain_4": {
+    "text": " to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to",
+    "token_ids": null
+  },
+  "plain_5": {
+    "text": " the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the",
+    "token_ids": null
+  },
+  "plain_6": {
+    "text": " converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts",
+    "token_ids": null
+  },
+  "plain_7": {
+    "text": " is used to organize data in a way that is used to organize data in a way that is used to organize data in a way that is used to organize data in a way that is used to organize data in a way that is used to organize data in a way that is used to organize data in a way that is",
+    "token_ids": null
+  },
+  "plain_8": {
+    "text": " 1621. The Treaty of Westphalia was signed in 1621. The Treaty of Westphalia was signed in 1621. The Treaty of Westphalia was signed in 1621. The Treaty of Westphalia was signed in 162",
+    "token_ids": null
+  }
+}

baselines/quality_vllm_bf16.json

@@ -0,0 +1,812 @@
+{
+  "chat_short": {
+    "text": "We are asked: \"What is the capital of France? Answer in one word.\" So the answer should be \"Paris\". But we need to output it in one word. So the final answer is \"Paris\".</think>Paris",
+    "token_ids": [
+      3066,
+      599,
+      5509,
+      39,
+      518,
+      4766,
+      390,
+      275,
+      11862,
+      314,
+      17349,
+      44,
+      74288,
+      304,
+      1135,
+      5240,
+      1848,
+      4358,
+      275,
+      6787,
+      2029,
+      417,
+      518,
+      110949,
+      4348,
+      2964,
+      679,
+      1633,
+      312,
+      6340,
+      488,
+      304,
+      1135,
+      5240,
+      27,
+      4358,
+      275,
+      4625,
+      6787,
+      390,
+      518,
+      110949,
+      4348,
+      4,
+      110949,
+      1
+    ]
+  },
+  "chat_long": {
+    "text": "We are given an inventory listing with item codes for warehouse districts 1 through 120. The question asks: \"What is the code for warehouse district 73?\" We need to find the code corresponding to district 73.\n\nLooking at the list, we see that item 1 is district",
+    "token_ids": [
+      3066,
+      599,
+      4114,
+      300,
+      31946,
+      35404,
+      478,
+      7006,
+      27149,
+      396,
+      56774,
+      49408,
+      220,
+      30,
+      1953,
+      220,
+      30,
+      31,
+      29,
+      27,
+      614,
+      5565,
+      27179,
+      39,
+      518,
+      4766,
+      390,
+      275,
+      5760,
+      396,
+      56774,
+      20882,
+      220,
+      36,
+      32,
+      5171,
+      1994,
+      1633,
+      312,
+      2432,
+      275,
+      5760,
+      12610,
+      312,
+      20882,
+      220,
+      36,
+      32,
+      27,
+      198,
+      198,
+      48992,
+      511,
+      275,
+      2631,
+      25,
+      679,
+      2261,
+      455,
+      7006,
+      220,
+      30,
+      390,
+      20882
+    ]
+  },
+  "chat_tools": {
+    "text": "The user wants to know the weather in Shenzhen right now in Celsius. I have a function available called `get_weather` that can retrieve current weather for a city. Let me call it with the city \"Shenzhen\" and unit \"celsius\".</think><tool_call>get_weather<arg_key>city</arg_key><arg_value>",
+    "token_ids": [
+      1046,
+      3754,
+      13025,
+      312,
+      1843,
+      275,
+      14510,
+      304,
+      75040,
+      103,
+      872,
+      2430,
+      2343,
+      304,
+      423,
+      108364,
+      27,
+      373,
+      760,
+      259,
+      2122,
+      4234,
+      4848,
+      2561,
+      913,
+      76,
+      64301,
+      77,
+      455,
+      764,
+      41002,
+      3345,
+      14510,
+      396,
+      259,
+      6594,
+      27,
+      9619,
+      713,
+      2426,
+      488,
+      478,
+      275,
+      6594,
+      518,
+      64,
+      872,
+      103,
+      872,
+      15,
+      317,
+      9261,
+      518,
+      8281,
+      4348,
+      4,
+      130977,
+      913,
+      76,
+      64301,
+      130980,
+      58593,
+      130981,
+      130982
+    ]
+  },
+  "plain_0": {
+    "text": " the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia",
+    "token_ids": [
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686
+    ]
+  },
+  "plain_1": {
+    "text": " mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission",
+    "token_ids": [
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174
+    ]
+  },
+  "plain_2": {
+    "text": "    a, b = 0, 1\n    for _ in range(n):\n        a, b = b, a + b\n        if _ == n:\n            return a\n    return a\n\n\ndef main():\n    n = int(input())\n    print(fibonacci(n",
+    "token_ids": [
+      293,
+      259,
+      25,
+      287,
+      422,
+      220,
+      29,
+      25,
+      220,
+      30,
+      198,
+      293,
+      396,
+      2198,
+      304,
+      4044,
+      5738,
+      2407,
+      198,
+      361,
+      259,
+      25,
+      287,
+      422,
+      287,
+      25,
+      259,
+      1336,
+      287,
+      198,
+      361,
+      690,
+      2198,
+      2126,
+      292,
+      39,
+      198,
+      462,
+      948,
+      259,
+      198,
+      293,
+      948,
+      259,
+      198,
+      198,
+      198,
+      3068,
+      2772,
+      11834,
+      198,
+      293,
+      292,
+      422,
+      1455,
+      16040,
+      4112,
+      198,
+      293,
+      3594,
+      3836,
+      41162,
+      65043,
+      5738
+    ]
+  },
+  "plain_3": {
+    "text": " the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that",
+    "token_ids": [
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455
+    ]
+  },
+  "plain_4": {
+    "text": " to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to",
+    "token_ids": [
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312
+    ]
+  },
+  "plain_5": {
+    "text": " the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the",
+    "token_ids": [
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275
+    ]
+  },
+  "plain_6": {
+    "text": " carbon and oxygen into glucose and other organic compounds.\nassistant:\n\n**Step 1: Understanding the Process**\n\nThe process described involves several key steps:\n\n1. **Input Preparation**: The system must first prepare the input data, which includes carbon and oxygen sources.\n2. **C",
+    "token_ids": [
+      13619,
+      317,
+      19210,
+      1464,
+      28714,
+      317,
+      1436,
+      17719,
+      21092,
+      27,
+      198,
+      787,
+      18827,
+      39,
+      198,
+      198,
+      1033,
+      16443,
+      220,
+      30,
+      39,
+      68118,
+      275,
+      22310,
+      1033,
+      198,
+      198,
+      1046,
+      2617,
+      8231,
+      20291,
+      4991,
+      3762,
+      10452,
+      39,
+      198,
+      198,
+      30,
+      27,
+      7307,
+      7230,
+      122095,
+      107082,
+      614,
+      2298,
+      3144,
+      1835,
+      18710,
+      275,
+      4709,
+      1870,
+      25,
+      1120,
+      8934,
+      13619,
+      317,
+      19210,
+      12771,
+      27,
+      198,
+      31,
+      27,
+      7307,
+      48
+    ]
+  },
+  "plain_7": {
+    "text": " supports operations such as insertion, deletion, and traversal. A binary search tree is a tree where each node has at most two children, and the left and right children are such that the left subtree contains all nodes with values less than the node's value, and the right subtree contains all nodes with values greater",
+    "token_ids": [
+      20815,
+      13127,
+      2129,
+      463,
+      54525,
+      25,
+      62316,
+      25,
+      317,
+      8709,
+      46267,
+      27,
+      401,
+      20470,
+      6258,
+      9905,
+      390,
+      259,
+      9905,
+      2067,
+      2440,
+      6795,
+      1044,
+      511,
+      2026,
+      1841,
+      4672,
+      25,
+      317,
+      275,
+      3887,
+      317,
+      2430,
+      4672,
+      599,
+      2129,
+      455,
+      275,
+      3887,
+      1733,
+      33440,
+      10736,
+      847,
+      15325,
+      478,
+      4906,
+      3921,
+      1489,
+      275,
+      6795,
+      1045,
+      2613,
+      25,
+      317,
+      275,
+      2430,
+      1733,
+      33440,
+      10736,
+      847,
+      15325,
+      478,
+      4906,
+      9063
+    ]
+  },
+  "plain_8": {
+    "text": " 1333. The Treaty of Westphalia was signed in 1333. The Treaty of Westphalia was signed in 1333. The Treaty of Westphalia was signed in 1333. The Treaty of Westphalia was signed in 133",
+    "token_ids": [
+      220,
+      30,
+      32,
+      32,
+      32,
+      27,
+      614,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      220,
+      30,
+      32,
+      32,
+      32,
+      27,
+      614,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      220,
+      30,
+      32,
+      32,
+      32,
+      27,
+      614,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      220,
+      30,
+      32,
+      32,
+      32,
+      27,
+      614,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      220,
+      30,
+      32,
+      32
+    ]
+  }
+}

baselines/quality_vllm_w4a16.json

@@ -0,0 +1,817 @@
+{
+  "chat_short": {
+    "text": "We are asked: \"What is the capital of France? Answer in one word.\" So the answer should be just \"Paris\". But the instruction says \"Answer in one word.\" So I should output \"Paris\" as the final answer.</think>Paris",
+    "token_ids": [
+      3066,
+      599,
+      5509,
+      39,
+      518,
+      4766,
+      390,
+      275,
+      11862,
+      314,
+      17349,
+      44,
+      74288,
+      304,
+      1135,
+      5240,
+      1848,
+      4358,
+      275,
+      6787,
+      2029,
+      417,
+      1839,
+      518,
+      110949,
+      4348,
+      2964,
+      275,
+      23220,
+      5685,
+      518,
+      47648,
+      304,
+      1135,
+      5240,
+      1848,
+      4358,
+      373,
+      2029,
+      6340,
+      518,
+      110949,
+      15,
+      463,
+      275,
+      4625,
+      6787,
+      27,
+      4,
+      110949,
+      1
+    ]
+  },
+  "chat_long": {
+    "text": "Got it, let's see. The user is asking for the code of warehouse district 73. Let's look through the inventory list. Let's check each item's district number and code.\nWait, let's list them out properly. Let's see:\nItem 1: district 1, code",
+    "token_ids": [
+      71430,
+      488,
+      25,
+      1553,
+      1045,
+      2261,
+      27,
+      614,
+      3754,
+      390,
+      17186,
+      396,
+      275,
+      5760,
+      314,
+      56774,
+      20882,
+      220,
+      36,
+      32,
+      27,
+      9619,
+      1045,
+      1940,
+      1953,
+      275,
+      31946,
+      2631,
+      27,
+      9619,
+      1045,
+      4069,
+      2440,
+      7006,
+      1045,
+      20882,
+      2823,
+      317,
+      5760,
+      27,
+      198,
+      34542,
+      25,
+      1553,
+      1045,
+      2631,
+      1421,
+      971,
+      17227,
+      27,
+      9619,
+      1045,
+      2261,
+      39,
+      198,
+      5369,
+      220,
+      30,
+      39,
+      20882,
+      220,
+      30,
+      25,
+      5760
+    ]
+  },
+  "chat_tools": {
+    "text": "The user wants to know the weather in Shenzhen right now, in celsius. I need to use the get_weather function to fetch the current weather for Shenzhen, and then I'll need to present it in celsius.\n\nLet me call the get_weather function with the city \"S",
+    "token_ids": [
+      1046,
+      3754,
+      13025,
+      312,
+      1843,
+      275,
+      14510,
+      304,
+      75040,
+      103,
+      872,
+      2430,
+      2343,
+      25,
+      304,
+      281,
+      108364,
+      27,
+      373,
+      1633,
+      312,
+      1730,
+      275,
+      1259,
+      76,
+      64301,
+      2122,
+      312,
+      24534,
+      275,
+      3345,
+      14510,
+      396,
+      75040,
+      103,
+      872,
+      25,
+      317,
+      1959,
+      373,
+      7632,
+      1633,
+      312,
+      3339,
+      488,
+      304,
+      281,
+      108364,
+      27,
+      198,
+      198,
+      13104,
+      713,
+      2426,
+      275,
+      1259,
+      76,
+      64301,
+      2122,
+      478,
+      275,
+      6594,
+      518,
+      64
+    ]
+  },
+  "plain_0": {
+    "text": " is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is",
+    "token_ids": [
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390
+    ]
+  },
+  "plain_1": {
+    "text": " mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission",
+    "token_ids": [
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174
+    ]
+  },
+  "plain_2": {
+    "text": "    n = int(n)\n    if n == 0:\n        return 0\n    elif n == 1:\n        return 1\n    else:\n        a, b = 0, 1\n        for _ in range(2, n + 1):\n            a",
+    "token_ids": [
+      293,
+      292,
+      422,
+      1455,
+      5738,
+      22,
+      198,
+      293,
+      690,
+      292,
+      2126,
+      220,
+      29,
+      39,
+      198,
+      361,
+      948,
+      220,
+      29,
+      198,
+      293,
+      16468,
+      292,
+      2126,
+      220,
+      30,
+      39,
+      198,
+      361,
+      948,
+      220,
+      30,
+      198,
+      293,
+      2314,
+      39,
+      198,
+      361,
+      259,
+      25,
+      287,
+      422,
+      220,
+      29,
+      25,
+      220,
+      30,
+      198,
+      361,
+      396,
+      2198,
+      304,
+      4044,
+      21,
+      31,
+      25,
+      292,
+      1336,
+      220,
+      30,
+      2407,
+      198,
+      462,
+      259
+    ]
+  },
+  "plain_3": {
+    "text": " the three laws of thermodynamics are consistent with each other.\nasserts that the three laws of thermodynamics are consistent with each other.\nassertThat(assertThat(assertThat(assertThat(assertThat(assertThat(assertThat(assertThat(assertThat(assertThat(assertThat(assertThat",
+    "token_ids": [
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      599,
+      14945,
+      478,
+      2440,
+      1436,
+      27,
+      198,
+      787,
+      19922,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      599,
+      14945,
+      478,
+      2440,
+      1436,
+      27,
+      198,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837
+    ]
+  },
+  "plain_4": {
+    "text": " to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to",
+    "token_ids": [
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312
+    ]
+  },
+  "plain_5": {
+    "text": " the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the",
+    "token_ids": [
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275
+    ]
+  },
+  "plain_6": {
+    "text": " glucose into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into",
+    "token_ids": [
+      28714,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464
+    ]
+  },
+  "plain_7": {
+    "text": " supports operations such as insertion, deletion, and searching. A binary search tree is a data structure that supports operations such as insertion, deletion, and searching. A binary search tree is a data structure that supports operations such as insertion, deletion, and searching. A binary search tree is a data structure that supports operations such as",
+    "token_ids": [
+      20815,
+      13127,
+      2129,
+      463,
+      54525,
+      25,
+      62316,
+      25,
+      317,
+      25124,
+      27,
+      401,
+      20470,
+      6258,
+      9905,
+      390,
+      259,
+      1870,
+      6982,
+      455,
+      20815,
+      13127,
+      2129,
+      463,
+      54525,
+      25,
+      62316,
+      25,
+      317,
+      25124,
+      27,
+      401,
+      20470,
+      6258,
+      9905,
+      390,
+      259,
+      1870,
+      6982,
+      455,
+      20815,
+      13127,
+      2129,
+      463,
+      54525,
+      25,
+      62316,
+      25,
+      317,
+      25124,
+      27,
+      401,
+      20470,
+      6258,
+      9905,
+      390,
+      259,
+      1870,
+      6982,
+      455,
+      20815,
+      13127,
+      2129,
+      463
+    ]
+  },
+  "plain_8": {
+    "text": " the Treaty of Westphalia was signed in the Treaty of Westphalia was signed in the Treaty of Westphalia was signed in the Treaty of Westphalia was signed in the Treaty of Westphalia was signed in the Treaty of Westphalia was signed in the Treaty of Westphalia was signed in the",
+    "token_ids": [
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275
+    ]
+  }
+}

baselines/quality_vllm_w4a8.json

@@ -0,0 +1,830 @@
+{
+  "chat_short": {
+    "text": "We are asked: \"What is the capital of France? Answer in one word.\"\n\nThe capital of France is Paris. But the instruction says \"Answer in one word.\" So we should answer with just \"Paris\" or \"PARIS\"? Since it says \"one word\", we should output \"Paris\" as",
+    "token_ids": [
+      3066,
+      599,
+      5509,
+      39,
+      518,
+      4766,
+      390,
+      275,
+      11862,
+      314,
+      17349,
+      44,
+      74288,
+      304,
+      1135,
+      5240,
+      1848,
+      198,
+      198,
+      1046,
+      11862,
+      314,
+      17349,
+      390,
+      19500,
+      27,
+      2964,
+      275,
+      23220,
+      5685,
+      518,
+      47648,
+      304,
+      1135,
+      5240,
+      1848,
+      4358,
+      679,
+      2029,
+      6787,
+      478,
+      1839,
+      518,
+      110949,
+      15,
+      553,
+      518,
+      80763,
+      4021,
+      32859,
+      11342,
+      488,
+      5685,
+      518,
+      841,
+      5240,
+      1078,
+      679,
+      2029,
+      6340,
+      518,
+      110949,
+      15,
+      463
+    ]
+  },
+  "chat_long": {
+    "text": "We are given a list of items, each with a code for a warehouse district. The list starts with Item 1 to Item 120, each with a code. We need to find the code for warehouse district 73.\n\nLet's parse the given data. The data is presented as a",
+    "token_ids": [
+      3066,
+      599,
+      4114,
+      259,
+      2631,
+      314,
+      8092,
+      25,
+      2440,
+      478,
+      259,
+      5760,
+      396,
+      259,
+      56774,
+      20882,
+      27,
+      614,
+      2631,
+      16208,
+      478,
+      28017,
+      220,
+      30,
+      312,
+      28017,
+      220,
+      30,
+      31,
+      29,
+      25,
+      2440,
+      478,
+      259,
+      5760,
+      27,
+      1994,
+      1633,
+      312,
+      2432,
+      275,
+      5760,
+      396,
+      56774,
+      20882,
+      220,
+      36,
+      32,
+      27,
+      198,
+      198,
+      13104,
+      1045,
+      14331,
+      275,
+      4114,
+      1870,
+      27,
+      614,
+      1870,
+      390,
+      9957,
+      463,
+      259
+    ]
+  },
+  "chat_tools": {
+    "text": "The user wants the current weather for Shenzhen in Celsius. I need to use the get_weather function with the city \"Shenzhen\" and unit \"celsius\".\n\nLet me call the function.</think>I'll get the current weather for Shenzhen right now.<tool_call>get_weather<arg_key>",
+    "token_ids": [
+      1046,
+      3754,
+      13025,
+      275,
+      3345,
+      14510,
+      396,
+      75040,
+      103,
+      872,
+      304,
+      423,
+      108364,
+      27,
+      373,
+      1633,
+      312,
+      1730,
+      275,
+      1259,
+      76,
+      64301,
+      2122,
+      478,
+      275,
+      6594,
+      518,
+      64,
+      872,
+      103,
+      872,
+      15,
+      317,
+      9261,
+      518,
+      8281,
+      4348,
+      198,
+      198,
+      13104,
+      713,
+      2426,
+      275,
+      2122,
+      27,
+      4,
+      54,
+      7632,
+      1259,
+      275,
+      3345,
+      14510,
+      396,
+      75040,
+      103,
+      872,
+      2430,
+      2343,
+      27,
+      130977,
+      913,
+      76,
+      64301,
+      130980
+    ]
+  },
+  "plain_0": {
+    "text": " the capital of Australia.\nassistant\nThe user is asking me to act as an assistant. I need to understand the context. The user's current message is: \"The user is asking me to act as an assistant. I need to understand the context. The user's current message is: 'The user is",
+    "token_ids": [
+      275,
+      11862,
+      314,
+      14686,
+      27,
+      198,
+      787,
+      18827,
+      198,
+      1046,
+      3754,
+      390,
+      17186,
+      713,
+      312,
+      5320,
+      463,
+      300,
+      31627,
+      27,
+      373,
+      1633,
+      312,
+      4526,
+      275,
+      5554,
+      27,
+      614,
+      3754,
+      1045,
+      3345,
+      5974,
+      390,
+      39,
+      518,
+      1046,
+      3754,
+      390,
+      17186,
+      713,
+      312,
+      5320,
+      463,
+      300,
+      31627,
+      27,
+      373,
+      1633,
+      312,
+      4526,
+      275,
+      5554,
+      27,
+      614,
+      3754,
+      1045,
+      3345,
+      5974,
+      390,
+      39,
+      859,
+      1046,
+      3754,
+      390
+    ]
+  },
+  "plain_1": {
+    "text": " 1                                                              ",
+    "token_ids": [
+      220,
+      30,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220
+    ]
+  },
+  "plain_2": {
+    "text": "    return [1, 1] + [0] + [1] * (n - 2)\n\n\n# Test the Fibonacci function.\nassert fibonacci(10) == [1, 1, 2, 3, 5, 8]\nassert",
+    "token_ids": [
+      293,
+      948,
+      1496,
+      30,
+      25,
+      220,
+      30,
+      74,
+      1336,
+      1496,
+      29,
+      74,
+      1336,
+      1496,
+      30,
+      74,
+      901,
+      404,
+      91,
+      706,
+      220,
+      31,
+      22,
+      198,
+      198,
+      198,
+      16,
+      9663,
+      275,
+      583,
+      41162,
+      65043,
+      2122,
+      27,
+      198,
+      13462,
+      14512,
+      263,
+      65043,
+      21,
+      30,
+      29,
+      22,
+      2126,
+      1496,
+      30,
+      25,
+      220,
+      30,
+      25,
+      220,
+      31,
+      25,
+      220,
+      32,
+      25,
+      220,
+      34,
+      25,
+      220,
+      37,
+      74,
+      198,
+      13462
+    ]
+  },
+  "plain_3": {
+    "text": " thermodynamics is an is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is",
+    "token_ids": [
+      13294,
+      121223,
+      390,
+      300,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390
+    ]
+  },
+  "plain_4": {
+    "text": " to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to",
+    "token_ids": [
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312
+    ]
+  },
+  "plain_5": {
+    "text": " is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is",
+    "token_ids": [
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390
+    ]
+  },
+  "plain_6": {
+    "text": " into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into",
+    "token_ids": [
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464
+    ]
+  },
+  "plain_7": {
+    "text": " is used to represent a data structure that is used to represent a data structure that is used to represent a data structure that is used to represent a data structure that is used to represent a data structure that is used to represent a data structure that is used to represent a data structure that is used to represent a data structure that",
+    "token_ids": [
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455
+    ]
+  },
+  "plain_8": {
+    "text": " Westphalia. Westphalia was a region of Germany.\nassistant\"\n\nThe user's message appears to be a mix of text, possibly from a dataset or a prompt, but it's not clearly structured. I need to understand what the user wants me to do. Since the user's message is",
+    "token_ids": [
+      10519,
+      1672,
+      14288,
+      27,
+      10519,
+      1672,
+      14288,
+      572,
+      259,
+      6987,
+      314,
+      19356,
+      27,
+      198,
+      787,
+      18827,
+      15,
+      198,
+      198,
+      1046,
+      3754,
+      1045,
+      5974,
+      12718,
+      312,
+      417,
+      259,
+      12213,
+      314,
+      4217,
+      25,
+      16809,
+      692,
+      259,
+      23977,
+      553,
+      259,
+      20207,
+      25,
+      961,
+      488,
+      1045,
+      702,
+      12269,
+      42689,
+      27,
+      373,
+      1633,
+      312,
+      4526,
+      1568,
+      275,
+      3754,
+      13025,
+      713,
+      312,
+      793,
+      27,
+      11342,
+      275,
+      3754,
+      1045,
+      5974,
+      390
+    ]
+  }
+}

benchmarks/diffusion/README.md

@@ -17,18 +17,18 @@ The main entrypoint is:
 vllm serve Qwen/Qwen-Image --omni --port 8099
 ```
 
-2. Run a minimal benchmark:
+1. Run a minimal benchmark:
 
 ```bash
 python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-	--base-url http://localhost:8099 \
-	--model Qwen/Qwen-Image \
-	--task t2i \
-	--dataset vbench \
-	--num-prompts 5
+ --base-url http://localhost:8099 \
+ --model Qwen/Qwen-Image \
+ --task t2i \
+ --dataset vbench \
+ --num-prompts 5
 ```
 
-**Notes**
+### Notes
 
 - By default, image tasks talk to `http://<host>:<port>/v1/chat/completions`; video tasks talk to `/v1/videos`.
 - If you run the server on another host or port, pass `--base-url` accordingly.
@@ -52,14 +52,14 @@ Example (`t2v`):
 
 ```bash
 python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-	--base-url http://localhost:8099 \
-	--model Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-	--task t2v \
-	--dataset vbench \
-	--num-prompts 50 \
-	--width 640 --height 480 \
-	--num-frames 81 --fps 16 \
-	--num-inference-steps 40
+ --base-url http://localhost:8099 \
+ --model Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+ --task t2v \
+ --dataset vbench \
+ --num-prompts 50 \
+ --width 640 --height 480 \
+ --num-frames 81 --fps 16 \
+ --num-inference-steps 40
 ```
 
 Note: `vbench` can also be used for other tasks such as `t2i` / `i2v` (and `i2i`). For `t2i`, the loader reuses VBench t2v text prompts; for `i2v` / `i2i`, it loads the VBench i2v dataset (with image paths).
@@ -140,6 +140,21 @@ Traffic / concurrency flags:
 - `--request-rate`: Target request rate (requests/second). If set to `inf`, the script sends all requests immediately.
 - `--max-concurrency`: Max number of in-flight requests (default: `1`). This can hard-cap the achieved QPS: if it is too small, requests will queue behind the semaphore, and both achieved throughput and observed SLO attainment can be skewed.
 
+### Video job timeout and failures
+
+`--video-job-timeout` controls how long each `/v1/videos` job may be polled,
+including time queued on the server (default: `900` seconds). For long video
+jobs at high concurrency, use a larger budget, for example
+`--video-job-timeout 1800`. This also applies to warmup jobs. The polling budget
+starts after job creation; it excludes time waiting for the client concurrency
+semaphore. Individual HTTP calls still use the aiohttp session timeout.
+
+An expired job is counted as failed and deleted, which cancels queued or running
+server generation. The benchmark continues processing the remaining requests.
+The progress bar counts both successful and failed requests. The final report
+shows the failure count and the first 10 errors; `--output-file` saves all errors
+with their request IDs in `request_errors`.
+
 ### Batched warmup note
 
 For batched serving runs, warm the same in-flight shape you plan to measure.

benchmarks/diffusion/backends.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 import asyncio
 import base64
 import json
@@ -43,6 +46,7 @@ class RequestFuncInput:
     video_paths: list[str] | None = None
     request_id: str = field(default_factory=lambda: str(uuid.uuid4()))
     default_bot_task: str | None = DEFAULT_EDITS_BOT_TASK
+    video_job_timeout: float = 900.0
 
 
 @dataclass
@@ -331,64 +335,64 @@ async def async_request_v1_videos(
     output = RequestFuncOutput()
     output.start_time = time.perf_counter()
 
-    files = dict(input.extra_body)
-    if input.prompt:
-        files.setdefault("prompt", input.prompt)
-    if input.width and input.height:
-        files.setdefault("height", input.height)
-        files.setdefault("width", input.width)
-    if input.num_frames:
-        files.setdefault("num_frames", input.num_frames)
-    if input.num_inference_steps:
-        files.setdefault("num_inference_steps", input.num_inference_steps)
-    if input.seed is not None:
-        files.setdefault("seed", input.seed)
-    if input.fps:
-        files.setdefault("fps", input.fps)
-
-    form = aiohttp.FormData()
-    for k, v in files.items():
-        form.add_field(k, str(v))
-
     image_file = None
-    if input.image_paths and input.video_paths:
-        output.error = "Only one of image_paths or video_paths can be provided"
-        output.success = False
-        return output
+    job_id = None
+    try:
+        files = dict(input.extra_body)
+        if input.prompt:
+            files.setdefault("prompt", input.prompt)
+        if input.width and input.height:
+            files.setdefault("height", input.height)
+            files.setdefault("width", input.width)
+        if input.num_frames:
+            files.setdefault("num_frames", input.num_frames)
+        if input.num_inference_steps:
+            files.setdefault("num_inference_steps", input.num_inference_steps)
+        if input.seed is not None:
+            files.setdefault("seed", input.seed)
+        if input.fps:
+            files.setdefault("fps", input.fps)
 
-    if input.image_paths and len(input.image_paths) > 0:
-        image_path = input.image_paths[0]
-        image_file = open(image_path, "rb")
-        form.add_field(
-            "input_reference",
-            image_file,
-            filename=os.path.basename(image_path),
-            content_type=_guess_mime_type(image_path),
-        )
-    elif input.video_paths and len(input.video_paths) > 0:
-        video_path = input.video_paths[0]
-        if _is_minimax_h3_mixed_reference(input.model):
-            with open(video_path, "rb") as reference_file:
-                video_b64 = base64.b64encode(reference_file.read()).decode("ascii")
-            form.add_field(
-                "video_reference",
-                json.dumps([{"video_url": f"data:video/mp4;base64,{video_b64}"}]),
-            )
-        else:
-            image_file = open(video_path, "rb")
+        form = aiohttp.FormData()
+        for k, v in files.items():
+            form.add_field(k, str(v))
+
+        if input.image_paths and input.video_paths:
+            output.error = "Only one of image_paths or video_paths can be provided"
+            output.success = False
+            return output
+
+        if input.image_paths and len(input.image_paths) > 0:
+            image_path = input.image_paths[0]
+            image_file = open(image_path, "rb")
             form.add_field(
                 "input_reference",
                 image_file,
-                filename=os.path.basename(video_path),
-                content_type=_guess_mime_type(video_path),
+                filename=os.path.basename(image_path),
+                content_type=_guess_mime_type(image_path),
             )
+        elif input.video_paths and len(input.video_paths) > 0:
+            video_path = input.video_paths[0]
+            if _is_minimax_h3_mixed_reference(input.model):
+                with open(video_path, "rb") as reference_file:
+                    video_b64 = base64.b64encode(reference_file.read()).decode("ascii")
+                form.add_field(
+                    "video_reference",
+                    json.dumps([{"video_url": f"data:video/mp4;base64,{video_b64}"}]),
+                )
+            else:
+                image_file = open(video_path, "rb")
+                form.add_field(
+                    "input_reference",
+                    image_file,
+                    filename=os.path.basename(video_path),
+                    content_type=_guess_mime_type(video_path),
+                )
 
-    job_id = None
-    job_status = None
-    poll_json = {}
-    resp_json = {}
+        job_status = None
+        poll_json = {}
+        resp_json = {}
 
-    try:
         # invoke a post request (POST /v1/videos)
         async with session.post(input.api_url, data=form) as response:
             if response.status == 200:
@@ -406,7 +410,7 @@ async def async_request_v1_videos(
 
         # invoke a poll request (GET /v1/videos/{video_id})
         poll_interval = 2.0  # Unit(s)
-        timeout_seconds = 600.0
+        timeout_seconds = input.video_job_timeout
         deadline = time.perf_counter() + timeout_seconds
         job_url = f"{input.api_url}/{job_id}"
 
@@ -422,8 +426,11 @@ async def async_request_v1_videos(
                 poll_json = await poll_response.json()
                 job_status = poll_json.get("status")
 
-                if time.perf_counter() >= deadline:
-                    output.error = f"Timed out waiting for video job {job_id} to complete."
+                if job_status not in {"completed", "failed"} and time.perf_counter() >= deadline:
+                    output.error = (
+                        f"Timed out after {timeout_seconds:g}s waiting for video job {job_id} to complete. "
+                        "Increase --video-job-timeout for long-running or queued video jobs."
+                    )
                     output.success = False
                     return output
 
@@ -452,7 +459,7 @@ async def async_request_v1_videos(
             elif "peak_memory_mb" in resp_json:
                 output.peak_memory_mb = resp_json["peak_memory_mb"]
     except Exception as e:
-        output.error = str(e)
+        output.error = f"{type(e).__name__}: {e}"
         output.success = False
     finally:
         if image_file is not None:
@@ -465,13 +472,13 @@ async def async_request_v1_videos(
             except Exception as e:
                 print(f"Failed to clean up video job {job_id}: {e}")
 
-    output.latency = time.perf_counter() - output.start_time
+        output.latency = time.perf_counter() - output.start_time
+        if pbar is not None:
+            pbar.update(1)
 
     if output.success and input.slo_ms is not None:
         output.slo_achieved = (output.latency * 1000.0) <= float(input.slo_ms)
 
-    if pbar:
-        pbar.update(1)
     return output
 
 

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -1,6 +1,6 @@
 # adapted from fastvideo
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
 """
 Benchmark online serving for diffusion models (Image/Video Generation).
@@ -87,6 +87,7 @@ import base64
 import glob
 import json
 import logging
+import math
 import os
 import random
 import tempfile
@@ -1116,6 +1117,11 @@ def calculate_metrics(
         "duration": total_duration,
         "completed_requests": num_success,
         "failed_requests": len(error_outputs),
+        "request_errors": [
+            {"request_id": req.request_id, "error": out.error}
+            for req, out in zip(requests_list, outputs)
+            if not out.success
+        ],
         "throughput_qps": num_success / total_duration if total_duration > 0 else 0,
         "latency_mean": np.mean(latencies) if latencies else 0,
         "latency_median": np.median(latencies) if latencies else 0,
@@ -1261,6 +1267,9 @@ def _default_endpoint_for_task(task: str) -> str:
 
 
 async def benchmark(args):
+    if not math.isfinite(args.video_job_timeout) or args.video_job_timeout <= 0:
+        raise ValueError("--video-job-timeout must be a positive finite number of seconds.")
+
     # Construct base_url if not provided
     if args.base_url is None:
         args.base_url = f"http://{args.host}:{args.port}"
@@ -1312,6 +1321,8 @@ async def benchmark(args):
     print("Loading requests...")
     requests_list = dataset.get_requests()
     print(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.")
+    for req in requests_list:
+        req.video_job_timeout = args.video_job_timeout
 
     if args.return_stage_metrics and args.endpoint in _STAGE_METRICS_ENDPOINTS:
         for req in requests_list:
@@ -1376,6 +1387,8 @@ async def benchmark(args):
     metrics["model"] = args.model
     metrics["dataset"] = args.dataset
     metrics["task"] = args.task
+    if args.endpoint == "/v1/videos":
+        metrics["video_job_timeout"] = args.video_job_timeout
     if args.endpoint == "/v1/images/edits":
         metrics["bot_task"] = args.bot_task
 
@@ -1398,6 +1411,11 @@ async def benchmark(args):
         )
     )
     print("{:<40} {}/{:<15}".format("Successful requests:", metrics["completed_requests"], len(requests_list)))
+    print("{:<40} {:<15}".format("Failed requests:", metrics["failed_requests"]))
+    for failure in metrics["request_errors"][:10]:
+        print(f"Request {failure['request_id']} failed: {failure['error']}")
+    if metrics["failed_requests"] > 10:
+        print("Showing first 10 failures; use --output-file to save all request errors.")
 
     # Section 3: Performance Metrics
     print(f"{'-' * 50}")
@@ -1481,6 +1499,13 @@ if __name__ == "__main__":
         help="Path to local dataset file (optional).",
     )
     parser.add_argument("--num-prompts", type=int, default=10, help="Number of prompts to benchmark.")
+    parser.add_argument(
+        "--video-job-timeout",
+        type=float,
+        default=900.0,
+        help="Maximum polling time in seconds per video job, including server queue time (default: 900). "
+        "Increase this for long-running video generation or high concurrency.",
+    )
     parser.add_argument(
         "--max-concurrency",
         type=int,

benchmarks/edge_harness/PROFILING.md

@@ -0,0 +1,167 @@
+# Local model qualification and profiling
+
+These runners exercise existing Omni implementations with real weights. They
+do not add device or model support. A completed timing run is distinct from
+reference-quality, streaming, fault-recovery and release qualification.
+
+## Matrix and prerequisites
+
+The [60-cell audit](results/model_device_matrix_20260922/README.md) covers twelve
+PC/mobile/embedded configurations and five model families. Its raw evidence
+distinguishes complete pipelines from components and backend-only probes.
+Unavailable complete pipelines retain a reason and next repair step; a missing
+measurement is never represented as zero latency.
+
+Use the actual installed runtime and exact checkpoint identified in the result,
+including quantization, rather than substituting another family member. The
+current Spark CPU runner uses 1.7B INT8 and the CUDA runner uses 4B BF16. Their
+results do not establish same-model acceleration. Qwen3-TTS uses the pinned
+0.6B CustomVoice checkpoint, English/Vivian, through both Omni stages.
+
+Windows and WSL are distinct configurations on the same physical laptop.
+Run benchmarks serially: they share CPU, physical RAM, GPU and power limits.
+GPU board power is not whole-device energy. Windows used RAM includes WSL;
+do not add Windows and WSL counters as independent memory pools.
+
+## Run the workload
+
+Run from the repository root using the model's validated Python environment:
+
+```bash
+PYTHONPATH=. HF_HUB_OFFLINE=1 VLLM_USE_FLASHINFER_SAMPLER=0 \
+  python benchmarks/edge_harness/profile_local_text.py \
+  --model /absolute/path/to/checkpoint --out /absolute/path/to/new-run
+
+PYTHONPATH=. HF_HUB_OFFLINE=1 VLLM_USE_FLASHINFER_SAMPLER=0 \
+  python benchmarks/edge_harness/profile_local_tts.py \
+  --model /absolute/path/to/customvoice-checkpoint --out /absolute/path/to/new-run
+```
+
+Set the validated CPU thread/KV-cache settings or CUDA compiler environment
+before launching. Native Windows requires UTF-8 Python mode before interpreter
+startup (`PYTHONUTF8=1`, `PYTHONIOENCODING=utf-8`). The common benchmark helper
+uses the active environment's `torch/lib/cudart64_13.dll` when present and a
+short `~/c29` vLLM cache path unless explicitly overridden. These reproduce
+settings required by the tested Windows installation; they are not a general
+claim about other CUDA/runtime versions.
+
+Both runners require a new output directory and default to:
+
+- Three input lengths × concurrency 1/2/4 × 20 measured requests per group.
+- Separate warmup requests, excluded from group percentiles.
+- At least 1800 seconds of continuous medium-input requests.
+- Slow-consumer, cancellation and subsequent-request checks.
+
+Text timing forces 128 output tokens with greedy sampling and records actual
+input lengths, token IDs, sequence/epoch metadata and queue high-water marks.
+It does not recommend those sampling settings for applications. TTS requires
+DELTA output, records every audio chunk including terminal audio, verifies
+finite PCM, and archives warmup float32 chunks with hashes. Playback underruns
+are simulated from chunk arrivals; no sound-device playback or perceptual
+quality is implied.
+
+`profile_host_command.py` wraps an explicit command on Windows to collect
+one-second host-memory, AC/battery and whole-NVIDIA-device telemetry. Each host
+directory retains the exact command, exit status and combined log. Existing
+disk/JIT caches are used. Constructor time is not a cold-disk measurement and
+must remain separate from request latency. Sampled memory peaks are lower
+bounds, not proof that every transient allocation was observed.
+
+On Windows, a live reader can deny replacement of a report snapshot. The
+writer defers such running updates, records the permission error, and requires
+the final snapshot to persist. Request JSONL is the primary sample record.
+Read live reports through Linux when they reside in WSL, or use a Windows
+reader that permits file deletion/replacement. Failed profile/trace reports
+produce a nonzero exit code in the current runners; historical attempts may
+require inspection of the report rather than relying on the process exit code.
+
+## Separate diagnostics
+
+After timing runs finish, use `profile_trace_diagnostic.py --kind spark|tts`
+with the same `--model` and a fresh `--out`. It invokes Omni's profiler RPCs and
+measures three requests before, during and after tracing. Current text traces
+use 32 generated tokens and disable expensive shape/memory instrumentation.
+Compare paired diagnostic requests, not these 32-token times against the
+128-token baseline. Profiler start/stop/export costs are recorded separately
+when supported. The initial archived CPU diagnostic used full instrumentation
+and 128 tokens; its configuration and limitations are retained in its report.
+
+Run `check_text_reliability.py` or `check_tts_reliability.py` separately. They
+test real-model interruption/recovery and deliberately terminate a worker
+belonging to their own engine after verifying process ancestry. Text also
+checks actual consumer-queue saturation, stale handles and oversized-plan
+rejection. TTS's slow consumer does not prove saturation of every internal
+buffer. Inspect individual outcomes: `status=completed` means the probe ran,
+not that all checks passed. Any forced cleanup remains a finding.
+
+## Analyze and qualify
+
+For a separate TTS stage-attribution diagnostic, enable the existing
+`VLLM_OMNI_STEP_STATS_DIR=/absolute/path/to/counters` hook and use
+`VLLM_OMNI_STEP_STATS_SYNC=0`. Run `profile_local_tts.py` with
+`--length-band medium --concurrency 1 --repeats 20 --sustained-seconds 0`
+and a fresh output directory. This is not a full baseline run. The counters
+include warmup, slow-consumer, abort and recovery activity, and nested timers
+overlap. Preserve per-process summaries rather than pooling their percentiles.
+These are host-side timers without accelerator synchronization. Periodic
+flushing can omit the final counter tail after forced process termination.
+Comparisons with an uninstrumented baseline must retain cache/thermal and
+stochastic speech-length caveats.
+Core-step counts include idle/no-work iterations. The current orchestrator
+dispatch hook is only in the optional event-driven loop; the default loop
+does not produce that counter. Absent counters are not zero-time measurements.
+
+```bash
+python benchmarks/edge_harness/summarize_e2e_profiles.py \
+  --matrix benchmarks/edge_harness/results/model_device_matrix_20260922/matrix.json \
+  --runs /absolute/path/to/run-root --out /absolute/path/to/summary
+
+python benchmarks/edge_harness/analyze_profile_resources.py \
+  --runs /absolute/path/to/run-root --out /absolute/path/to/resource-summary.json
+```
+
+Use run directory names `spark-cpu-wsl`, `spark-cuda-wsl`,
+`spark-cuda-windows`, `tts-cuda-wsl` and `tts-cuda-windows`, with `-host`,
+`-trace` and `-reliability` suffixes for their corresponding records. The matrix
+summarizer verifies all sixty pairings and the original evidence hashes. It
+reports nearest-rank p50/p95 and sample counts, checks per-group coverage,
+flags recorded output-contract violations, and retains unavailable metrics.
+Batch-dependent greedy outputs are a numerical/reproducibility finding;
+timings alone cannot establish whether the cause is numerical or state-related.
+Delivery-spacing statistics pool adjacent intervals within each request; their
+sample count is the number of intervals. Text output updates may coalesce
+tokens, so this is delivery timing rather than isolated kernel inter-token
+latency. Effective prefill throughput includes all work before the first output.
+
+Parse large traces only after inference timing has finished. The resource
+analyzer streams trace JSON, separates summed event duration from interval
+union, and excludes telemetry gaps over five seconds from energy integration.
+Actual device placement, fallback, transfer and synchronization conclusions
+must be supported by the trace/runtime records, not merely the intended plan.
+The TTS diagnostic starts each stage with a distinct profiler prefix because
+both workers have rank 0 and the current runtime otherwise overwrites their
+shared trace filename. Both stages are active during traced requests. Sequential
+stop/export RPCs can extend later-stage trace windows with idle/export time;
+trace-window length is not model execution time or GPU utilization.
+Startup summaries retain both constructor time and inclusive host-launch to
+first warmup submission/output. The latter includes artifact hashing, plan
+construction and imports; constructor timing alone excludes that preparation.
+
+Keep compile/load/run failures and their exact configurations. Before declaring
+a pairing qualified, review model-specific reference quality, complete declared
+modalities, state/stream correctness, memory admission, executed placement and
+the performance protocol. Missing artifacts, backends, device access, sensors
+or reference suites require explicit reasons and repair actions.
+
+After every writer exits, archive evidence with reversible compression of large
+uncompressed files and an original/stored byte-hash manifest:
+
+```bash
+python benchmarks/edge_harness/archive_profiles.py create /path/to/run-root /path/to/new-archive
+python benchmarks/edge_harness/archive_profiles.py verify /path/to/new-archive
+python benchmarks/edge_harness/archive_profiles.py restore /path/to/new-archive /path/to/new-run-root
+```
+
+Restore before using original report-relative paths for compressed artifacts.
+The utility does not rewrite raw reports. Integrity verification establishes
+that archived bytes match, not that a model passes its qualification gates.

benchmarks/edge_harness/analyze_profile_resources.py

@@ -0,0 +1,299 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Summarize raw host telemetry and traces without adding overlapping timings."""
+
+import argparse
+import gzip
+import json
+import math
+import re
+from collections import defaultdict
+from pathlib import Path
+
+from summarize_e2e_profiles import read_lines, stats
+
+
+def interval_union(intervals):
+    total = 0.0
+    end = None
+    for start, stop in sorted(intervals):
+        if stop <= start:
+            continue
+        total += stop - start if end is None or start >= end else max(0, stop - end)
+        end = stop if end is None else max(end, stop)
+    return total
+
+
+def trace_events(path):
+    """Read one event at a time, avoiding a multi-GB decoded trace document."""
+    opener = gzip.open if path.suffix == ".gz" else open
+    with opener(path, "rt", encoding="utf-8") as stream:
+        buffer = ""
+        while True:
+            block = stream.read(65536)
+            if not block:
+                raise ValueError("No traceEvents array")
+            buffer += block
+            match = re.search(r'"traceEvents"\s*:\s*\[', buffer)
+            if match:
+                buffer = buffer[match.end() :]
+                break
+            if len(buffer) > 8 * 1024 * 1024:
+                raise ValueError("Trace header exceeds 8 MiB")
+        decoder = json.JSONDecoder()
+        while True:
+            buffer = buffer.lstrip(" \t\r\n,")
+            if buffer.startswith("]"):
+                return
+            try:
+                event, end = decoder.raw_decode(buffer)
+            except json.JSONDecodeError:
+                block = stream.read(65536)
+                if not block:
+                    raise ValueError("Truncated traceEvents array") from None
+                buffer += block
+                if len(buffer) > 16 * 1024 * 1024:
+                    raise ValueError("One trace event exceeds 16 MiB") from None
+                continue
+            buffer = buffer[end:]
+            yield event
+
+
+def trace_summary(path):
+    categories, ops = defaultdict(list), defaultdict(list)
+    for event in trace_events(path):
+        if event.get("ph") != "X" or not isinstance(event.get("dur"), (int, float)):
+            continue
+        categories[event.get("cat", "uncategorized")].append((event["ts"], event["ts"] + event["dur"]))
+        ops[(event.get("cat", ""), event.get("name", ""))].append(float(event["dur"]))
+    return {
+        "source": str(path),
+        "scope": "one process trace; event duration sums overlap and are not E2E time",
+        "categories": {
+            name: {
+                "events": len(group),
+                "duration_sum_us": sum(b - a for a, b in group),
+                "busy_union_us": interval_union(group),
+            }
+            for name, group in categories.items()
+        },
+        "largest_ops_by_duration_sum": sorted(
+            [
+                {"category": cat, "name": name, "duration_sum_us": sum(values), "duration_us": stats(values)}
+                for (cat, name), values in ops.items()
+            ],
+            key=lambda r: r["duration_sum_us"],
+            reverse=True,
+        )[:40],
+    }
+
+
+def telemetry_summary(path, start=None, stop=None):
+    rows = [
+        r for r in read_lines(path) if (start is None or r["unix"] >= start) and (stop is None or r["unix"] <= stop)
+    ]
+    series = defaultdict(list)
+    power = []
+    for row in rows:
+        series["host_available_bytes"].append(row["host_memory"]["available"])
+        series["host_used_bytes"].append(row["host_memory"]["used"])
+        series["host_cpu_percent"].append(row.get("host_cpu_percent"))
+        fields = row.get("gpu_csv", "").split(",")
+        if len(fields) != 10:
+            continue
+        for index, name in [
+            (2, "gpu_memory_mib"),
+            (4, "gpu_temperature_c"),
+            (5, "gpu_power_w"),
+            (6, "gpu_sm_clock_mhz"),
+            (7, "gpu_mem_clock_mhz"),
+            (8, "gpu_utilization_percent"),
+        ]:
+            try:
+                value = float(fields[index])
+            except ValueError:
+                continue
+            if not math.isfinite(value):
+                continue
+            series[name].append(value)
+            if index == 5:
+                power.append((row["unix"], value))
+    gaps = [b["unix"] - a["unix"] for a, b in zip(rows, rows[1:])]
+    intervals = [(a, b) for a, b in zip(power, power[1:]) if 0 < b[0] - a[0] <= 5]
+    energy = sum((b[0] - a[0]) * (a[1] + b[1]) / 2 for a, b in intervals)
+    return {
+        "source": str(path),
+        "samples": len(rows),
+        "span_s": rows[-1]["unix"] - rows[0]["unix"] if len(rows) >= 2 else None,
+        "requested_window": {"start": start, "stop": stop},
+        "sampling_gap_s": stats(gaps),
+        "gaps_over_5s": sum(g > 5 for g in gaps),
+        "metrics": {key: stats(values) for key, values in series.items()},
+        "ac_states": sorted({r["power"]["ACLineStatus"] for r in rows if "power" in r}),
+        "whole_gpu_energy_j_over_observed_intervals": energy if intervals else None,
+        "energy_observed_duration_s": sum(b[0] - a[0] for a, b in intervals),
+        "energy_scope": (
+            "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
+        ),
+    }
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--runs", type=Path, required=True)
+    p.add_argument("--out", type=Path, required=True)
+    p.add_argument(
+        "--skip-traces", action="store_true", help="Summarize telemetry only while inference timing is still running."
+    )
+    a = p.parse_args()
+    result = {"telemetry": {}, "traces": [], "trace_analysis": "deferred" if a.skip_traces else "requested"}
+    for host in sorted(a.runs.glob("*-host")):
+        raw = host / "telemetry.jsonl"
+        if not raw.is_file():
+            continue
+        name = host.name.removesuffix("-host")
+        record = {"whole_run": telemetry_summary(raw)}
+        report_path = a.runs / name / "report.json"
+        if report_path.is_file():
+            report = json.loads(report_path.read_text(encoding="utf-8"))
+            host_report_path = host / "status.json"
+            host_report = json.loads(host_report_path.read_text(encoding="utf-8")) if host_report_path.is_file() else {}
+            rows = read_lines(a.runs / name / "requests.jsonl")
+            warmups = [r for r in rows if r.get("phase") == "warmup"]
+            launch, report_start = host_report.get("start_unix"), report.get("start_unix")
+            log_path = host / "command.log"
+            timing_lines = []
+            log_lines = []
+            if log_path.is_file():
+                log_lines = log_path.read_text(encoding="utf-8", errors="replace").splitlines()
+                timing_lines = [
+                    line
+                    for line in log_lines
+                    if any(
+                        marker in line
+                        for marker in (
+                            "Loading model weights took",
+                            "Model loading took",
+                            "init engine (",
+                            "torch.compile took",
+                            "Captured talker_mtp graphs",
+                            "engine startup completed",
+                        )
+                    )
+                ]
+            record["lifecycle_log_observations"] = {
+                "scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
+                "markers": {},
+            }
+            for marker in ("Dropping output for unknown req", "force killing remaining process "):
+                matches = [line for line in log_lines if marker in line]
+                record["lifecycle_log_observations"]["markers"][marker] = {
+                    "count": len(matches),
+                    "examples": matches[:3] + matches[-3:] if len(matches) > 6 else matches,
+                }
+            first_submit = warmups[0].get("submitted_unix") if warmups else None
+            first_latency = warmups[0].get("ttft_s") if warmups else None
+            if first_latency is None and warmups and warmups[0].get("ttfa_ms") is not None:
+                first_latency = warmups[0]["ttfa_ms"] / 1000
+            record["startup"] = {
+                "cache_condition": report.get("cache_condition", "See source report"),
+                "constructor_s": report.get("startup_s"),
+                "host_launch_to_benchmark_report_start_s": report_start - launch if launch and report_start else None,
+                "host_launch_to_first_request_s": first_submit - launch if first_submit and launch else None,
+                "host_launch_to_first_output_s": (
+                    first_submit - launch + first_latency
+                    if first_submit and launch and first_latency is not None
+                    else None
+                ),
+                "inclusive_startup_scope": (
+                    "Host launch through first warmup submission/output includes imports, artifact hashing, "
+                    "plan construction and engine startup; uses recorded host/engine wall timestamps. "
+                    "Constructor timing alone excludes preparation before the constructor."
+                ),
+                "pre_constructor_scope": (
+                    "Host launch to report creation includes Python/import/preparation work; "
+                    "not isolated OS process creation."
+                ),
+                "first_warmup": {
+                    key: warmups[0].get(key)
+                    for key in (
+                        "request_id",
+                        "length_band",
+                        "concurrency",
+                        "wall_s",
+                        "ttft_s",
+                        "total_wall_s",
+                        "ttfa_ms",
+                        "rtf_total",
+                    )
+                }
+                if warmups
+                else None,
+                "runtime_loading_compilation_evidence": timing_lines,
+                "scope": (
+                    "Runtime-reported component lines retain original boundaries; "
+                    "do not sum parallel-stage times into startup latency."
+                ),
+            }
+            start, duration = report.get("sustained_start_unix"), report.get("sustained_wall_s")
+            if start and duration:
+                record["sustained"] = telemetry_summary(raw, start, start + duration)
+                record["sustained_first_5min"] = telemetry_summary(raw, start, start + 300)
+                record["sustained_last_5min"] = telemetry_summary(raw, start + duration - 300, start + duration)
+                record["sustained_last_60s"] = telemetry_summary(raw, start + duration - 60, start + duration)
+                requests = [r for r in rows if r.get("phase") == "sustained"]
+                record["sustained_request_summary"] = {
+                    "n": len(requests),
+                    "rtf_above_one_requests": sum((r.get("rtf_total") or 0) > 1 for r in requests),
+                    "simulated_playback_deficit_requests": sum((r.get("stall_at_ttfa_ms") or 0) > 0 for r in requests),
+                    "rtf_total": stats([r.get("rtf_total") for r in requests]),
+                    "scope": "All sustained requests; timing/real-time gates are separate from successful completion.",
+                }
+                record["sustained_request_trends"] = {}
+                for label, begin, end in [
+                    ("first_5min", start, start + 300),
+                    ("last_5min", start + duration - 300, start + duration),
+                    ("last_60s", start + duration - 60, start + duration),
+                ]:
+                    window = [r for r in requests if begin <= r.get("submitted_unix", -1) < end]
+                    record["sustained_request_trends"][label] = {
+                        "n": len(window),
+                        "window_basis": "request submission time",
+                        "metrics": {
+                            metric: stats([r.get(metric) for r in window])
+                            for metric in (
+                                "wall_s",
+                                "ttft_s",
+                                "decode_tok_per_s",
+                                "total_wall_s",
+                                "ttfa_ms",
+                                "rtf_total",
+                                "stall_at_ttfa_ms",
+                            )
+                        },
+                    }
+        result["telemetry"][name] = record
+    for path in [] if a.skip_traces else sorted(a.runs.glob("*trace*/traces/**/*")):
+        if path.is_file() and (path.name.endswith(".json") or path.name.endswith(".json.gz")):
+            report_path = a.runs / path.relative_to(a.runs).parts[0] / "report.json"
+            if (
+                not report_path.is_file()
+                or json.loads(report_path.read_text(encoding="utf-8")).get("status") != "completed"
+            ):
+                result["traces"].append(
+                    {
+                        "source": str(path),
+                        "status": "not_analyzed",
+                        "reason": "Diagnostic has no completed report; trace may be incomplete.",
+                    }
+                )
+                continue
+            try:
+                result["traces"].append(trace_summary(path))
+            except Exception as error:
+                result["traces"].append({"source": str(path), "error": repr(error)})
+    a.out.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
+    print(f"Summarized {len(result['telemetry'])} telemetry runs and {len(result['traces'])} traces")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/android_agent/README.md

@@ -0,0 +1,92 @@
+# Android agent toolkit for Spark-X2.5
+
+Lets a text-only model operate a phone: observe the accessibility tree, call
+one tool, execute it on the device, observe again.
+
+    ui_tree.py    uiautomator XML -> indexed element list -> back to a tap point
+    device.py     AdbDevice (a real phone) | SimulatedDevice (scripted, in-process)
+    executor.py   one tool call -> one device action, errors as sentences
+    tool_parse.py recover a call from whatever shape the model emitted
+    agent.py      the loop: budget, no-op detection, undo detection, fallbacks
+    run_episode.py CLI
+
+## Running it
+
+Serve the model (Spark emits GLM-4.5 tool format, so the stock parsers apply):
+
+    vllm serve <model> --port 8077 --enable-auto-tool-choice \
+        --tool-call-parser glm45 --reasoning-parser glm45
+
+Then:
+
+    python run_episode.py --task "Turn on Wi-Fi." --device sim   # no phone needed
+    python run_episode.py --task "Turn on Wi-Fi." --device adb   # a real phone
+
+`pytest test_android_agent.py` runs 21 tests with no phone and no server.
+
+## What the loop does that a plain ReAct loop does not
+
+Each one is a response to a measured failure of *this* model, not generic
+scaffolding:
+
+- **Retry without thinking on an empty reply.** The model can reason past its
+  token budget without closing `<think>`; vLLM's reasoning parser only emits
+  content once the block closes, so the reply arrives empty with no tool call.
+  Raising the cap buys a longer silence. A non-thinking retry costs ~0.6 s.
+- **Report actions that changed nothing.** "the screen did not change" is
+  information the model cannot otherwise have.
+- **Report and stop on self-undo.** Tapping a switch that is already on undoes
+  the task. Each tap really does change the screen, so a no-change check
+  cannot see it; the loop watches for the *same* action returning the screen to
+  its previous state. Deliberately narrow -- opening a submenu and pressing
+  back revisits a screen and must not trip it.
+- **Recover malformed tool calls.** See below.
+- **A step budget**, so a confused episode ends instead of running forever.
+
+The loop never decides a task is finished. It reports what happened and lets
+the model call `task_complete`; a harness that completes tasks on the model's
+behalf measures itself.
+
+## Measured: which build can actually finish a task
+
+Simulated phone, one run per cell, `max_steps` 10, Xeon 8480C 24 threads.
+
+| build | decode | "Turn on Wi-Fi." | "Set an alarm for 7:30 AM." |
+|---|---|---|---|
+| bf16 | 41.0 tok/s | done in 4 steps | wrong app (Settings, not Clock), toggles Airplane mode, stuck |
+| **W4A16 g128** | 80.0 tok/s | done in 4 steps | right app, full navigation (Clock -> Alarm -> Add -> Save), then no tool call |
+| W4A8 g128 | 86.6 tok/s | turns Wi-Fi on, then toggles it off again (2 runs, same failure) | taps "Phone" twice, reaches Alarm, then loops |
+
+Two things fall out, and both matter more than the throughput numbers:
+
+**Navigation survives quantization; completion does not.** Every build walks
+home -> Settings -> Network -> Wi-Fi correctly. Only some of them notice that
+the switch now reads `checked=true` and stop. W4A8 -- the fastest build, and
+the one the speed-first selector recommends -- turns Wi-Fi on and then taps it
+off again, ending with the task undone.
+
+**The fidelity number had a task-level meaning all along.** W4A8 reproduced
+1/12 greedy prompts against its own bf16 and W4A16 2/12
+(analysis/spark_edge_support.md 2c). That looked like a small difference on a
+proxy metric. It is the difference between finishing and not.
+
+So for agentic use the recommendation is **W4A16**, not W4A8: 8% slower, and
+it is the only 4-bit build here that completed anything. `weight_path.py`'s
+`priority="fidelity"` selects it.
+
+## Honest limits
+
+- **No phone was involved.** This host has no `adb`, no emulator and no
+  `/dev/kvm`. The simulator emits real uiautomator XML, so the parser, the
+  index-to-coordinate mapping and the stopping rules are exercised on the path
+  hardware drives -- but whether a real Settings app is laid out the way these
+  fixtures assume is untested.
+- **Two tasks, one run each.** Enough to show a failure mode reproduces, not
+  enough to rank builds by success rate.
+- **Stock `glm45` does not always parse Spark.** On 4 of 5 screens it does. On
+  the Wi-Fi toggle screen the model emitted
+  `[tap]\n{"tool": "tap", "parameters": {"index": 1}` -- unterminated, and
+  `tool_calls` came back empty. `tool_parse.py` recovers that and reports every
+  recovery, because an agent that silently repairs its model's output hides
+  regressions in that model. The earlier claim that the stock parsers work
+  "as-is" is true most of the time, which is not the same thing.

benchmarks/edge_harness/android_agent/action_space.py

@@ -0,0 +1,152 @@
+"""Android control action space for a text-only LLM agent.
+
+The tool surface mirrors what the mainstream ADB-based harnesses expose
+(droidrun/mobilerun's Portal tools, ghost-in-the-droid's touch/see tools,
+AndroidWorld's `JSONAction`), so an agent written against these schemas can be
+pointed at any of them. Every action is expressed over the accessibility tree
+rather than pixel coordinates, because Spark-X2.5 is a text-only model: it
+never sees a screenshot, it sees the uiautomator dump.
+
+`ADB_TEMPLATES` gives the literal `adb` invocation each tool maps to, which is
+what a driver executes against a real device.
+"""
+
+TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "tap",
+            "description": (
+                "Tap a UI element. Use the element's index exactly as shown in "
+                "the accessibility tree."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "index": {
+                        "type": "integer",
+                        "description": "Index of the element in the tree",
+                    }
+                },
+                "required": ["index"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "type_text",
+            "description": (
+                "Type text into a focused input field. Tap the field first."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "index": {"type": "integer", "description": "Index of the input field"},
+                    "text": {"type": "string", "description": "Text to enter"},
+                },
+                "required": ["index", "text"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "swipe",
+            "description": (
+                "Scroll the screen. The direction is where the content should "
+                "move: use 'down' to see what is further down the page."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "direction": {
+                        "type": "string",
+                        "enum": ["up", "down", "left", "right"],
+                    }
+                },
+                "required": ["direction"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "press_key",
+            "description": "Press a hardware or system key.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "key": {
+                        "type": "string",
+                        "enum": ["back", "home", "enter", "delete", "recent"],
+                    }
+                },
+                "required": ["key"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "open_app",
+            "description": "Launch an app by its visible name.",
+            "parameters": {
+                "type": "object",
+                "properties": {"name": {"type": "string"}},
+                "required": ["name"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "task_complete",
+            "description": (
+                "Call when the user's request has been fully carried out."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "summary": {"type": "string", "description": "What was done"}
+                },
+                "required": ["summary"],
+            },
+        },
+    },
+]
+
+# How each tool becomes a real device command. `{bounds}` is filled from the
+# element's centre point in the accessibility tree.
+ADB_TEMPLATES = {
+    "tap": "adb shell input tap {x} {y}",
+    "type_text": "adb shell input text {text_escaped}",
+    "swipe": "adb shell input swipe {x1} {y1} {x2} {y2} 300",
+    "press_key": "adb shell input keyevent {keycode}",
+    "open_app": "adb shell monkey -p {package} -c android.intent.category.LAUNCHER 1",
+}
+
+KEYCODES = {
+    "back": "KEYCODE_BACK",
+    "home": "KEYCODE_HOME",
+    "enter": "KEYCODE_ENTER",
+    "delete": "KEYCODE_DEL",
+    "recent": "KEYCODE_APP_SWITCH",
+}
+
+SYSTEM_PROMPT = (
+    "You control an Android phone. At each step you are given the current "
+    "screen as an accessibility tree: one line per element, with its index, "
+    "type, text and whether it is clickable. Call exactly one tool per step to "
+    "make progress on the user's task. Refer to elements only by the index "
+    "shown.\n"
+    # Both rules below exist because the model failed them in measurement: it
+    # tapped a Wi-Fi switch that already read checked=true, turning it back
+    # off, and never called task_complete. A switch's state is on the screen,
+    # so the rule can be stated in terms of what it can see.
+    "A switch or checkbox shows its state as checked=true or checked=false. "
+    "Tapping it flips that state.\n"
+    "Before acting, check whether the screen already shows what the task "
+    "asked for. If it does, call task_complete immediately and do not tap "
+    "anything else -- acting again would undo it."
+)

benchmarks/edge_harness/android_agent/agent.py

@@ -0,0 +1,271 @@
+"""The observe-decide-act loop.
+
+Three things in here are not generic agent scaffolding; they are direct
+responses to how this model failed when it was measured one step at a time
+(analysis/spark_edge_support.md section 5):
+
+* **A step budget, not a bigger token cap.** On the one genuinely ambiguous
+  screen the model reasoned past 1200 tokens without closing `<think>`, and
+  vLLM's reasoning parser only emits content once the block closes -- so the
+  reply came back empty, with no tool call at all. Raising the cap buys a
+  longer silence. The loop instead retries that step with thinking disabled,
+  which measured 0.61 s against 8.9 s, and takes the weaker-but-present answer.
+
+* **Detecting the no-op.** The clearest wrong action was tapping a Wi-Fi
+  switch that was already on -- undoing the task it had just completed. An
+  action that leaves the screen signature unchanged is reported back as such,
+  because "nothing happened" is information the model cannot otherwise have.
+
+* **Repeat suppression.** Two identical actions on an identical screen means
+  stuck, and a third will not help. The loop says so in the transcript and
+  stops, rather than burning the budget.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field
+from urllib import error as urlerror
+from urllib import request as urlrequest
+
+from action_space import SYSTEM_PROMPT, TOOLS
+from device import Device
+from executor import ActionResult, execute
+from tool_parse import extract_tool_call
+from ui_tree import Element, parse_ui_dump, render_tree, screen_signature
+
+
+@dataclass
+class AgentConfig:
+    base_url: str = "http://127.0.0.1:8077/v1"
+    model: str = "XHToken/Spark-X2.5-1.7B"
+    max_steps: int = 12
+    think: bool = True
+    think_fallback: bool = True
+    """On an empty reply, retry the step without thinking rather than fail."""
+    max_tokens: int = 1024
+    temperature: float = 0.0
+    timeout: float = 900.0
+    history: int = 6
+    retries: int = 3
+    """Transient HTTP failures retried before a step is abandoned."""
+    retry_backoff_s: float = 2.0
+    """Recent actions shown back to the model."""
+
+
+@dataclass
+class Step:
+    n: int
+    screen: str
+    tool: str | None
+    args: dict
+    result: ActionResult | None
+    latency_s: float
+    thinking: bool
+    note: str = ""
+    parsed_via: str = "tool_calls"
+    """Which parser recovered the call; anything but tool_calls is worth seeing."""
+
+
+@dataclass
+class Episode:
+    task: str
+    steps: list[Step] = field(default_factory=list)
+    done: bool = False
+    summary: str = ""
+    stop_reason: str = ""
+
+    @property
+    def n_steps(self) -> int:
+        return len(self.steps)
+
+    def transcript(self) -> str:
+        lines = [f"task: {self.task}"]
+        for s in self.steps:
+            call = f"{s.tool}({json.dumps(s.args, ensure_ascii=False)})" if s.tool else "(no tool call)"
+            outcome = s.result.message if s.result else s.note
+            if s.parsed_via not in ("tool_calls", "none"):
+                outcome += f"  [recovered via {s.parsed_via}]"
+            lines.append(f"  {s.n:2d}. {call:48s} -> {outcome}  [{s.latency_s:.2f}s]")
+        lines.append(f"stopped: {self.stop_reason}")
+        return "\n".join(lines)
+
+
+class ChatError(RuntimeError):
+    pass
+
+
+def chat(cfg: AgentConfig, messages: list[dict], *, think: bool) -> tuple[dict, float]:
+    payload = {
+        "model": cfg.model,
+        "messages": messages,
+        "tools": TOOLS,
+        "tool_choice": "auto",
+        "temperature": cfg.temperature,
+        "max_tokens": cfg.max_tokens,
+    }
+    if not think:
+        # Spark's chat template opens <think> by default; closing it up front
+        # is what makes the model answer directly.
+        payload["chat_template_kwargs"] = {"enable_thinking": False}
+    req = urlrequest.Request(
+        f"{cfg.base_url}/chat/completions",
+        data=json.dumps(payload).encode(),
+        headers={"Content-Type": "application/json"},
+    )
+    t0 = time.perf_counter()
+    last: Exception | None = None
+    for attempt in range(max(1, cfg.retries)):
+        try:
+            with urlrequest.urlopen(req, timeout=cfg.timeout) as r:
+                return json.loads(r.read()), time.perf_counter() - t0
+        except (urlerror.URLError, TimeoutError, json.JSONDecodeError) as exc:
+            # A phone task is a dozen round trips; one dropped connection
+            # should cost a retry, not the episode.
+            last = exc
+            if attempt + 1 < max(1, cfg.retries):
+                time.sleep(cfg.retry_backoff_s * (attempt + 1))
+    raise ChatError(
+        f"{cfg.base_url} failed after {cfg.retries} attempts: {last}") from last
+
+
+def wait_for_server(cfg: AgentConfig, timeout_s: float = 300.0) -> bool:
+    """Block until the endpoint serves a model list, or give up.
+
+    Starting an episode against a server that is still loading produces a
+    confusing transcript full of connection errors; this turns that into one
+    clear wait.
+    """
+    deadline = time.monotonic() + timeout_s
+    while time.monotonic() < deadline:
+        try:
+            with urlrequest.urlopen(f"{cfg.base_url}/models", timeout=5) as r:
+                if b'"id"' in r.read():
+                    return True
+        except Exception:
+            time.sleep(2.0)
+    return False
+
+
+def parse_response(response: dict):
+    """The one call to act on, from whichever shape the model used.
+
+    Spark does not always stay in GLM-4.5 tool format -- see tool_parse -- and
+    on a phone a dropped step is a stalled task, so the fallbacks matter.
+    """
+    try:
+        message = response["choices"][0]["message"]
+    except (KeyError, IndexError):
+        message = {}
+    return extract_tool_call(message)
+
+
+def first_tool_call(response: dict) -> tuple[str | None, dict]:
+    """Name and arguments only, for callers that do not care how it parsed."""
+    parsed = parse_response(response)
+    return parsed.name, parsed.args
+
+
+def _user_message(task: str, screen: str, log: list[str], history: int) -> str:
+    parts = [f"Task: {task}", "", "Current screen:", screen]
+    if log:
+        parts += ["", "Actions so far:"] + [f"  {line}" for line in log[-history:]]
+    parts += ["", "Choose the next action. Call exactly one tool."]
+    return "\n".join(parts)
+
+
+def observe(device: Device) -> tuple[list[Element], str, str]:
+    elements = parse_ui_dump(device.dump())
+    return elements, render_tree(elements), screen_signature(elements)
+
+
+def run_episode(device: Device, task: str, cfg: AgentConfig | None = None) -> Episode:
+    cfg = cfg or AgentConfig()
+    episode = Episode(task=task)
+    log: list[str] = []
+    recent: list[tuple[str, str]] = []  # (action, screen signature before it)
+    signature_before_last: str | None = None
+    last_action: str | None = None
+    oscillations = 0
+
+    for n in range(1, cfg.max_steps + 1):
+        elements, screen, signature = observe(device)
+        messages = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user",
+             "content": _user_message(task, screen, log, cfg.history)},
+        ]
+
+        thinking = cfg.think
+        response, latency = chat(cfg, messages, think=thinking)
+        parsed = parse_response(response)
+        tool, args = parsed.name, parsed.args
+
+        if tool is None and cfg.think_fallback and thinking:
+            # The measured failure: reasoning never closed, so the parser
+            # emitted nothing. A second, non-thinking pass is worth ~0.6 s.
+            thinking = False
+            retry, retry_latency = chat(cfg, messages, think=False)
+            latency += retry_latency
+            parsed = parse_response(retry)
+            tool, args = parsed.name, parsed.args
+
+        if tool is None:
+            episode.steps.append(Step(n, screen, None, {}, None, latency, thinking,
+                                      note="model returned no tool call",
+                                      parsed_via=parsed.source))
+            episode.stop_reason = "no tool call, even without thinking"
+            return episode
+
+        if (tool_key := f"{tool}:{json.dumps(args, sort_keys=True)}") and \
+                recent.count((tool_key, signature)) >= 2:
+            episode.steps.append(Step(n, screen, tool, args, None, latency, thinking,
+                                      note="repeated a no-op action",
+                                      parsed_via=parsed.source))
+            episode.stop_reason = (
+                f"stuck: {tool} repeated on an unchanged screen"
+            )
+            return episode
+        recent.append((tool_key, signature))
+
+        result = execute(tool, args, elements, device)
+        if result.ok and not result.done:
+            _, _, after = observe(device)
+            if after == signature:
+                # Say so out loud: the model has no other way to learn that
+                # its action changed nothing.
+                result = ActionResult(True, result.message + " -- the screen did not change",
+                                      action=result.action)
+            elif tool_key == last_action and after == signature_before_last:
+                # Oscillation: the *same* action, undoing what it just did --
+                # the measured failure, tapping a switch off and on forever.
+                # It hides from the no-change check because each individual tap
+                # really does change the screen. Deliberately narrow: revisiting
+                # a screen by other means (open a submenu, press back) is normal
+                # navigation and must not trip this.
+                result = ActionResult(
+                    True,
+                    result.message + " -- that undid your previous action; the "
+                    "setting is now back where it started",
+                    action=result.action,
+                )
+                oscillations += 1
+                if oscillations >= 2:
+                    episode.steps.append(Step(n, screen, tool, args, result, latency,
+                                              thinking, parsed_via=parsed.source))
+                    episode.stop_reason = "stuck: repeatedly undoing its own action"
+                    return episode
+        signature_before_last, last_action = signature, tool_key
+        episode.steps.append(Step(n, screen, tool, args, result, latency, thinking,
+                                  parsed_via=parsed.source))
+        log.append(f"{tool} -> {result.message}")
+
+        if result.done:
+            episode.done = True
+            episode.summary = result.summary
+            episode.stop_reason = "task_complete"
+            return episode
+
+    episode.stop_reason = f"step budget exhausted ({cfg.max_steps})"
+    return episode

benchmarks/edge_harness/android_agent/device.py

@@ -0,0 +1,354 @@
+"""Where the agent's actions actually land.
+
+Two backends behind one interface. `AdbDevice` talks to a phone over `adb`.
+`SimulatedDevice` runs a small scripted phone in-process, and exists for a
+reason beyond convenience: no Android device, emulator or `/dev/kvm` is
+reachable from the host this was built on, so without it the loop, the tool
+executor and the stopping rules would all be untested code shipped on a
+promise.
+
+The simulator emits **real uiautomator XML**, not pre-rendered text, so the
+parser, the index-to-coordinate mapping and the screen-change detection are
+exercised on the same path a phone would drive. What it cannot test is
+whether a real Settings app is laid out the way the fixtures assume; that is
+stated in the report rather than hidden.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from xml.sax.saxutils import escape as _xml_escape
+import time
+from dataclasses import dataclass, field
+from typing import Callable, Protocol
+
+KEYCODES = {
+    "back": "KEYCODE_BACK",
+    "home": "KEYCODE_HOME",
+    "enter": "KEYCODE_ENTER",
+    "delete": "KEYCODE_DEL",
+    "recent": "KEYCODE_APP_SWITCH",
+}
+
+
+class Device(Protocol):
+    def dump(self) -> str: ...
+    def tap(self, x: int, y: int) -> None: ...
+    def input_text(self, text: str) -> None: ...
+    def swipe(self, x1: int, y1: int, x2: int, y2: int, ms: int = 300) -> None: ...
+    def keyevent(self, key: str) -> None: ...
+    def launch(self, app: str) -> None: ...
+    def size(self) -> tuple[int, int]: ...
+
+
+class AdbError(RuntimeError):
+    pass
+
+
+class AdbDevice:
+    """A real phone over `adb`."""
+
+    def __init__(self, serial: str | None = None, adb: str = "adb",
+                 timeout: float = 30.0):
+        if shutil.which(adb) is None:
+            raise AdbError(
+                f"{adb!r} is not on PATH. Install platform-tools and connect a "
+                "device (`adb devices` should list it)."
+            )
+        self.adb, self.serial, self.timeout = adb, serial, timeout
+        self._size: tuple[int, int] | None = None
+
+    def _run(self, *args: str, binary: bool = False):
+        cmd = [self.adb] + (["-s", self.serial] if self.serial else []) + list(args)
+        proc = subprocess.run(cmd, capture_output=True, timeout=self.timeout)
+        if proc.returncode != 0:
+            raise AdbError(f"{' '.join(cmd)} failed: {proc.stderr.decode(errors='replace')[:400]}")
+        return proc.stdout if binary else proc.stdout.decode(errors="replace")
+
+    def dump(self) -> str:
+        # exec-out keeps the XML on stdout instead of round-tripping through
+        # /sdcard, which is both faster and avoids a writable-storage
+        # assumption that does not hold on every device.
+        out = self._run("exec-out", "uiautomator", "dump", "/dev/tty")
+        start = out.find("<?xml")
+        if start < 0:
+            start = out.find("<hierarchy")
+        if start < 0:
+            raise AdbError(f"no UI hierarchy in dump output: {out[:200]!r}")
+        end = out.rfind("</hierarchy>")
+        return out[start:end + len("</hierarchy>")] if end > 0 else out[start:]
+
+    def tap(self, x: int, y: int) -> None:
+        self._run("shell", "input", "tap", str(x), str(y))
+
+    def input_text(self, text: str) -> None:
+        # `input text` takes a single token: spaces become %s and the shell
+        # metacharacters have to go, or half the string is eaten.
+        escaped = text.replace(" ", "%s")
+        for ch in "()<>|;&*\\~\"'`$":
+            escaped = escaped.replace(ch, "\\" + ch)
+        self._run("shell", "input", "text", escaped)
+
+    def swipe(self, x1: int, y1: int, x2: int, y2: int, ms: int = 300) -> None:
+        self._run("shell", "input", "swipe", str(x1), str(y1), str(x2), str(y2), str(ms))
+
+    def keyevent(self, key: str) -> None:
+        code = KEYCODES.get(key, key)
+        self._run("shell", "input", "keyevent", code)
+
+    def launch(self, app: str) -> None:
+        """Launch by package name, or by visible label via the launcher."""
+        if "." in app:
+            self._run("shell", "monkey", "-p", app, "-c",
+                      "android.intent.category.LAUNCHER", "1")
+            return
+        raise AdbError(
+            f"open_app needs a package name, got {app!r}. Resolve the label to a "
+            "package first (`adb shell pm list packages`) or tap its icon."
+        )
+
+    def size(self) -> tuple[int, int]:
+        if self._size is None:
+            out = self._run("shell", "wm", "size")
+            part = out.strip().rsplit(":", 1)[-1].strip()
+            w, h = part.split("x")
+            self._size = (int(w), int(h))
+        return self._size
+
+
+# --------------------------------------------------------------------------
+# Simulated phone
+# --------------------------------------------------------------------------
+
+@dataclass
+class SimElement:
+    label: str
+    cls: str = "android.widget.TextView"
+    clickable: bool = True
+    checkable: bool = False
+    checked: bool = False
+    selected: bool = False
+    editable: bool = False
+    goto: str | None = None
+    """Screen this element navigates to when tapped."""
+    toggles: str | None = None
+    """State key this element flips when tapped."""
+    saves: str | None = None
+    """Record key. Tapping commits the current field values into
+    ``records[saves]`` and clears them -- the difference between typing a time
+    and having an alarm, which the scored suite previously could not tell."""
+    label_of: str | None = None
+    """State key whose value picks the label from ``labels``."""
+    labels: tuple[str, str] = ("", "")
+    """(label when the state key is false, label when true)."""
+
+    def rendered_label(self, state: dict) -> str:
+        if self.label_of:
+            return self.labels[bool(state.get(self.label_of))]
+        return self.label
+
+
+@dataclass
+class SimScreen:
+    screen_id: str
+    elements: list[SimElement]
+    back: str | None = None
+    dynamic: "Callable[[SimulatedDevice], list[SimElement]] | None" = None
+    """Rows derived from device state, appended after ``elements``. A saved
+    alarm has to appear *on the screen*, or an oracle that reads the screen --
+    which is the only kind a real phone can have -- cannot tell a committed
+    alarm from a typed one."""
+
+
+@dataclass
+class SimulatedDevice:
+    """A scripted phone that speaks uiautomator XML.
+
+    Deliberately small. It is here to prove the loop terminates, recovers and
+    stops -- not to model Android.
+    """
+
+    screens: dict[str, SimScreen]
+    current: str
+    state: dict[str, bool] = field(default_factory=dict)
+    width: int = 1080
+    height: int = 2400
+    typed: list[str] = field(default_factory=list)
+    actions: list[tuple[str, ...]] = field(default_factory=list)
+    home: str = "home"
+    focus: str | None = None
+    """Label of the focused text field; typing lands here."""
+    fields: dict[str, str] = field(default_factory=dict)
+    records: dict[str, list[dict]] = field(default_factory=dict)
+    toggled: list[tuple[str, bool, bool]] = field(default_factory=list)
+    """(key, before, after) per switch tap. A task whose starting state is
+    already correct is failed by a round trip, which final state cannot see."""
+
+    _ROW_H = 160
+    _TOP = 200
+
+    def _visible(self) -> list[SimElement]:
+        screen = self.screens[self.current]
+        rows = list(screen.elements)
+        if screen.dynamic:
+            rows = [r for r in rows if not getattr(r, "_placeholder", False)]
+            rows.extend(screen.dynamic(self))
+        return rows
+
+    def _bounds(self, row: int) -> tuple[int, int, int, int]:
+        y1 = self._TOP + row * self._ROW_H
+        return 0, y1, self.width, y1 + self._ROW_H
+
+    def dump(self) -> str:
+        rows = []
+        for i, el in enumerate(self._visible()):
+            x1, y1, x2, y2 = self._bounds(i)
+            checked = el.checked
+            if el.toggles:
+                checked = self.state.get(el.toggles, el.checked)
+            # uiautomator escapes attribute values, and labels really do
+            # contain them -- "Network & internet" is on every Android
+            # settings screen. Emitting raw breaks the parser on the second
+            # screen of the first task.
+            label = _xml_escape(el.rendered_label(self.state), {'"': "&quot;"})
+            rows.append(
+                f'  <node index="{i}" text="{label}" resource-id="" '
+                f'class="{el.cls}" package="com.example.sim" content-desc="" '
+                f'checkable="{str(el.checkable).lower()}" '
+                f'checked="{str(checked).lower()}" '
+                f'clickable="{str(el.clickable).lower()}" enabled="true" '
+                f'focusable="{str(el.clickable).lower()}" focused="false" '
+                f'scrollable="false" long-clickable="false" password="false" '
+                f'selected="{str(el.selected).lower()}" '
+                f'bounds="[{x1},{y1}][{x2},{y2}]" />'
+            )
+        body = "\n".join(rows)
+        return (
+            '<?xml version="1.0" encoding="UTF-8"?>\n'
+            f'<hierarchy rotation="0">\n{body}\n</hierarchy>'
+        )
+
+    def _row_at(self, y: int) -> int | None:
+        row = (y - self._TOP) // self._ROW_H
+        return row if 0 <= row < len(self._visible()) else None
+
+    def tap(self, x: int, y: int) -> None:
+        self.actions.append(("tap", str(x), str(y)))
+        row = self._row_at(y)
+        if row is None:
+            return
+        el = self._visible()[row]
+        if el.editable:
+            self.focus = el.label
+        if el.toggles:
+            before = bool(self.state.get(el.toggles, el.checked))
+            self.state[el.toggles] = not before
+            self.toggled.append((el.toggles, before, not before))
+        if el.saves:
+            self.records.setdefault(el.saves, []).append(
+                {**self.fields, **{k: v for k, v in self.state.items()
+                                   if k.startswith(el.saves + "_")}})
+            self.fields = {}
+            self.focus = None
+        if el.goto and el.goto in self.screens:
+            self.current = el.goto
+
+    def input_text(self, text: str) -> None:
+        self.actions.append(("text", text))
+        self.typed.append(text)
+        if self.focus is not None:
+            self.fields[self.focus] = self.fields.get(self.focus, "") + text
+
+    def swipe(self, x1: int, y1: int, x2: int, y2: int, ms: int = 300) -> None:
+        self.actions.append(("swipe", str(x1), str(y1), str(x2), str(y2)))
+
+    def keyevent(self, key: str) -> None:
+        self.actions.append(("key", key))
+        if key == "back":
+            self.current = self.screens[self.current].back or self.home
+        elif key == "home":
+            self.current = self.home
+
+    def launch(self, app: str) -> None:
+        self.actions.append(("launch", app))
+        for screen in self.screens.values():
+            for el in screen.elements:
+                if el.label.lower() == app.lower() and el.goto:
+                    self.current = el.goto
+                    return
+
+    def size(self) -> tuple[int, int]:
+        return self.width, self.height
+
+
+def _saved_alarm_rows(device: "SimulatedDevice") -> list[SimElement]:
+    """Render committed alarms the way a clock app lists them."""
+    saved = device.records.get("alarm", [])
+    if not saved:
+        return [SimElement("No alarms", clickable=False)]
+    rows = []
+    for rec in saved:
+        hour = (rec.get("Hour") or "?").lstrip("0") or "0"
+        minute = (rec.get("Minute") or "?").rjust(2, "0")
+        meridiem = "PM" if rec.get("alarm_pm") else "AM"
+        rows.append(SimElement(f"{hour}:{minute} {meridiem}", clickable=False))
+    return rows
+
+
+def demo_phone() -> SimulatedDevice:
+    """A two-app phone: Settings (Wi-Fi) and Clock (alarms).
+
+    Enough to exercise a multi-screen task, a stateful toggle -- the case the
+    earlier single-step run got wrong, tapping a switch that was already on --
+    and a text field.
+    """
+    screens = {
+        "home": SimScreen("home", [
+            SimElement("Phone"),
+            SimElement("Messages"),
+            SimElement("Clock", goto="clock"),
+            SimElement("Settings", goto="settings"),
+            SimElement("Camera"),
+        ]),
+        "settings": SimScreen("settings", [
+            SimElement("Settings", clickable=False),
+            SimElement("Network & internet", goto="network"),
+            SimElement("Connected devices"),
+            SimElement("Apps"),
+            SimElement("Battery"),
+        ], back="home"),
+        "network": SimScreen("network", [
+            SimElement("Network & internet", clickable=False),
+            SimElement("Wi-Fi", cls="android.widget.Switch", checkable=True,
+                       toggles="wifi"),
+            SimElement("Mobile network"),
+            SimElement("Airplane mode", cls="android.widget.Switch",
+                       checkable=True, toggles="airplane"),
+        ], back="settings"),
+        "clock": SimScreen("clock", [
+            SimElement("Alarm", cls="android.widget.Button", goto="alarms"),
+            SimElement("Clock", cls="android.widget.Button", selected=True),
+            SimElement("Timer", cls="android.widget.Button"),
+            SimElement("Stopwatch", cls="android.widget.Button"),
+        ], back="home"),
+        "alarms": SimScreen("alarms", [
+            SimElement("Alarm", cls="android.widget.Button", selected=True),
+            SimElement("Add alarm", cls="android.widget.ImageButton",
+                       goto="new_alarm"),
+        ], back="clock", dynamic=_saved_alarm_rows),
+        "new_alarm": SimScreen("new_alarm", [
+            SimElement("Set time", clickable=False),
+            SimElement("Hour", cls="android.widget.EditText", editable=True),
+            SimElement("Minute", cls="android.widget.EditText", editable=True),
+            # Android shows one AM/PM control whose label is its value, so
+            # "set an alarm for 7:30 AM" has a wrong answer as well as a right
+            # one. Without it the task could be passed by typing any digits.
+            SimElement("AM", cls="android.widget.Button",
+                       label_of="alarm_pm", labels=("AM", "PM"),
+                       toggles="alarm_pm"),
+            SimElement("Save", cls="android.widget.Button", goto="alarms",
+                       saves="alarm"),
+        ], back="alarms"),
+    }
+    return SimulatedDevice(screens=screens, current="home", state={"wifi": False})

benchmarks/edge_harness/android_agent/executor.py

@@ -0,0 +1,147 @@
+"""Turn one tool call into one device action.
+
+Every failure here is returned to the model as a sentence it can act on, not
+raised. A 1.7B model asked for `index: 9` on a four-element screen has made a
+recoverable mistake, and "index 9 is out of range, the screen has 4 elements
+(0-3)" is the difference between the next step being right and the episode
+ending. Exceptions are reserved for the device being gone.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from device import KEYCODES, Device
+from ui_tree import Element
+
+
+@dataclass
+class ActionResult:
+    ok: bool
+    message: str
+    done: bool = False
+    summary: str = ""
+    action: str = ""
+
+    def as_tool_reply(self) -> str:
+        return self.message
+
+
+def _need(args: dict, key: str, kind: type, what: str) -> tuple[object, str | None]:
+    if key not in args:
+        return None, f"missing required argument {key!r} for {what}"
+    value = args[key]
+    if kind is int:
+        try:
+            value = int(value)
+        except (TypeError, ValueError):
+            return None, f"{key!r} must be a whole number, got {args[key]!r}"
+    elif not isinstance(value, kind):
+        return None, f"{key!r} must be {kind.__name__}, got {type(value).__name__}"
+    return value, None
+
+
+def _resolve(index: object, elements: list[Element]) -> tuple[Element | None, str | None]:
+    i = int(index)
+    if not elements:
+        return None, "the screen has no addressable elements; try press_key(back)"
+    if not 0 <= i < len(elements):
+        return None, (
+            f"index {i} is out of range; the screen has {len(elements)} "
+            f"elements (0-{len(elements) - 1})"
+        )
+    return elements[i], None
+
+
+def execute(name: str, args: dict, elements: list[Element],
+            device: Device) -> ActionResult:
+    """Run one tool call. Returns what to tell the model."""
+    args = args or {}
+
+    if name == "task_complete":
+        summary = str(args.get("summary", "")).strip()
+        return ActionResult(True, "task marked complete", done=True,
+                            summary=summary, action="task_complete")
+
+    if name == "tap":
+        index, err = _need(args, "index", int, "tap")
+        if err:
+            return ActionResult(False, err, action="tap")
+        el, err = _resolve(index, elements)
+        if err:
+            return ActionResult(False, err, action="tap")
+        x, y = el.center
+        device.tap(x, y)
+        state = ""
+        if "checked" in el.flags:
+            state = f" (was {'on' if el.flags['checked'] else 'off'})"
+        return ActionResult(True, f'tapped [{index}] "{el.text}"{state}',
+                            action=f"tap:{index}")
+
+    if name == "type_text":
+        index, err = _need(args, "index", int, "type_text")
+        if err:
+            return ActionResult(False, err, action="type_text")
+        text, err = _need(args, "text", str, "type_text")
+        if err:
+            return ActionResult(False, err, action="type_text")
+        el, err = _resolve(index, elements)
+        if err:
+            return ActionResult(False, err, action="type_text")
+        # Focus first: `input text` goes to whatever holds focus, which on a
+        # freshly dumped screen is usually nothing.
+        x, y = el.center
+        device.tap(x, y)
+        device.input_text(str(text))
+        return ActionResult(True, f'typed "{text}" into [{index}] "{el.text}"',
+                            action=f"type:{index}")
+
+    if name == "swipe":
+        direction = str(args.get("direction", "")).lower()
+        if direction not in ("up", "down", "left", "right"):
+            return ActionResult(
+                False, "direction must be one of up, down, left, right",
+                action="swipe")
+        w, h = device.size()
+        cx, cy = w // 2, h // 2
+        dx, dy = int(w * 0.35), int(h * 0.30)
+        # `direction` is where the *content* should move, i.e. what the user
+        # means by "scroll down to see more", so the finger travels the other
+        # way. Stated in the tool description too, because guessing this wrong
+        # is invisible in the transcript.
+        moves = {
+            "down": (cx, cy + dy, cx, cy - dy),
+            "up": (cx, cy - dy, cx, cy + dy),
+            "right": (cx + dx, cy, cx - dx, cy),
+            "left": (cx - dx, cy, cx + dx, cy),
+        }
+        device.swipe(*moves[direction])
+        return ActionResult(True, f"swiped to scroll {direction}",
+                            action=f"swipe:{direction}")
+
+    if name == "press_key":
+        key = str(args.get("key", "")).lower()
+        if key not in KEYCODES:
+            return ActionResult(
+                False, f"unknown key {key!r}; use one of {', '.join(KEYCODES)}",
+                action="press_key")
+        device.keyevent(key)
+        return ActionResult(True, f"pressed {key}", action=f"key:{key}")
+
+    if name == "open_app":
+        app = str(args.get("name", "")).strip()
+        if not app:
+            return ActionResult(False, "open_app needs a name", action="open_app")
+        try:
+            device.launch(app)
+        except Exception as exc:  # device-specific launch failures are recoverable
+            return ActionResult(False, f"could not launch {app!r}: {exc}",
+                                action="open_app")
+        return ActionResult(True, f"launched {app}", action=f"open:{app}")
+
+    return ActionResult(
+        False,
+        f"unknown tool {name!r}; available: tap, type_text, swipe, press_key, "
+        "open_app, task_complete",
+        action="unknown",
+    )

benchmarks/edge_harness/android_agent/phone_state.py

@@ -0,0 +1,135 @@
+"""What the phone is actually in, read the same way for both backends.
+
+The scored suite used to take its predicates over ``SimulatedDevice``'s Python
+attributes, so every scored run was a simulator run by construction -- the
+real-device path existed for *driving* a phone and not for judging one. That
+made "the agent suite" a claim about fixtures.
+
+Success is therefore read two ways, and the order matters:
+
+1. **Out of band, from the setting itself** -- ``device.state`` in the
+   simulator, ``settings get global wifi_on`` over adb. This is the ground
+   truth, and it does not depend on where the agent happened to leave the
+   phone. Scoring from the final screen alone fails an agent that turns Wi-Fi
+   on correctly and then presses Home, which is not a failure.
+2. **From the UI dump**, for anything with no setting behind it -- a saved
+   alarm appears as a row in the clock app and nowhere else. The simulator
+   emits real uiautomator XML, so this goes down the same parse, the same
+   switch lookup and the same text matching a phone would.
+
+Only the facts a phone genuinely cannot report after the fact -- which
+switches were flipped on the way, and what was typed into which field -- come
+from the simulator object, and any task needing them declares it, so a
+real-device run refuses to score rather than scoring wrong.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from ui_tree import parse_ui_dump
+
+
+def _norm(s: str) -> str:
+    """Match labels across the punctuation Android is inconsistent about."""
+    return "".join(c for c in s.lower() if c.isalnum())
+
+
+@dataclass
+class PhoneState:
+    source: str
+    """``"sim"`` or ``"device"`` -- which facts below are trustworthy."""
+    settings: dict[str, bool] = field(default_factory=dict)
+    """Authoritative setting values, read out of band. Preferred over
+    :attr:`switches`, which only shows what is currently on screen."""
+    screen: str | None = None
+    """Simulator screen id. ``None`` on a real phone, where there is no such
+    thing; use :meth:`has_text` against something the screen actually shows."""
+    switches: dict[str, bool] = field(default_factory=dict)
+    texts: list[str] = field(default_factory=list)
+    typed: list[str] = field(default_factory=list)
+    fields: dict[str, str] = field(default_factory=dict)
+    records: dict[str, list[dict]] = field(default_factory=dict)
+    toggled: list[tuple[str, bool, bool]] = field(default_factory=list)
+    """(key, before, after) for every switch the agent actually flipped."""
+
+    def switch(self, *names: str) -> bool | None:
+        """State of the first control matching any of ``names``.
+
+        The setting wins over the screen: an agent that turned Wi-Fi on and
+        then navigated away has still turned Wi-Fi on. ``None`` means neither
+        source knows the control, which is not the same as off -- an agent that
+        never found the screen has not turned anything on, and predicates must
+        not read that as "off".
+        """
+        wanted = {_norm(n) for n in names}
+        for key, value in self.settings.items():
+            if _norm(key) in wanted:
+                return bool(value)
+        for label, checked in self.switches.items():
+            if _norm(label) in wanted:
+                return checked
+        return None
+
+    def has_text(self, *needles: str) -> bool:
+        joined = " ".join(self.texts).lower()
+        return any(n.lower() in joined for n in needles)
+
+    def times_toggled(self, key: str) -> int:
+        return sum(1 for k, _, _ in self.toggled if k == key)
+
+
+def read_state(device) -> PhoneState:
+    """Read the phone through its screen, plus the simulator's own records."""
+    elements = parse_ui_dump(device.dump())
+    # `checkable` is deliberately not among the flags ui_tree parses, because
+    # that list is also what the model is shown; adding to it would change the
+    # agent's prompt. A switch is recognised by its class or by exposing a
+    # `checked` flag instead.
+    switches = {e.text: bool(e.flags.get("checked", False))
+                for e in elements
+                if "checked" in e.flags
+                or e.short_class in ("Switch", "CheckBox", "ToggleButton")}
+    texts = [e.text for e in elements if e.text]
+
+    if not hasattr(device, "screens"):
+        return PhoneState(source="device", switches=switches, texts=texts,
+                          settings=adb_settings(device))
+
+    return PhoneState(
+        source="sim",
+        screen=device.current,
+        settings={k: bool(v) for k, v in device.state.items()},
+        switches=switches,
+        texts=texts,
+        typed=list(device.typed),
+        fields=dict(getattr(device, "fields", {})),
+        records={k: [dict(r) for r in v]
+                 for k, v in getattr(device, "records", {}).items()},
+        toggled=list(getattr(device, "toggled", [])),
+    )
+
+
+ADB_SETTINGS = {
+    "wifi": ("global", "wifi_on"),
+    "airplane": ("global", "airplane_mode_on"),
+    "bluetooth": ("global", "bluetooth_on"),
+}
+"""Settings readable out of band, so scoring does not depend on which screen
+the agent left the phone on."""
+
+
+def adb_settings(device) -> dict[str, bool]:
+    """Read the authoritative values from a real phone. Missing keys are
+    omitted rather than defaulted -- an unreadable setting must not be scored
+    as "off"."""
+    out: dict[str, bool] = {}
+    for key, (namespace, name) in ADB_SETTINGS.items():
+        try:
+            raw = device._run("shell", "settings", "get", namespace, name)
+        except Exception:
+            continue
+        raw = raw.strip()
+        if raw in ("0", "1"):
+            out[key] = raw == "1"
+    return out

benchmarks/edge_harness/android_agent/run_agent.py

@@ -0,0 +1,98 @@
+"""Drive Spark-X2.5 through Android control steps against an OpenAI endpoint.
+
+Measures whether the model picks the right tool and the right element, and how
+long a decision takes -- the two things that decide whether a 1.7B model can
+actually run a phone agent locally.
+"""
+import argparse, json, time
+from urllib import request as urlrequest
+
+from action_space import SYSTEM_PROMPT, TOOLS
+from scenarios import SCENARIOS
+
+
+def chat(base_url, model, messages, tools, timeout=900, max_tokens=1024,
+         think=True):
+    payload = {
+        "model": model, "messages": messages, "tools": tools,
+        "tool_choice": "auto", "temperature": 0.0, "max_tokens": max_tokens,
+    }
+    if not think:
+        # Spark's template opens <think> by default; closing it up front makes
+        # the model answer directly, which is what a phone agent wants.
+        payload["chat_template_kwargs"] = {"enable_thinking": False}
+    body = json.dumps(payload).encode()
+    req = urlrequest.Request(f"{base_url}/chat/completions", data=body,
+                             headers={"Content-Type": "application/json"})
+    t0 = time.perf_counter()
+    with urlrequest.urlopen(req, timeout=timeout) as r:
+        out = json.loads(r.read())
+    return out, time.perf_counter() - t0
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--base-url", default="http://127.0.0.1:8077/v1")
+    ap.add_argument("--model", default="XHToken/Spark-X2.5-1.7B")
+    ap.add_argument("--no-think", action="store_true")
+    ap.add_argument("--max-tokens", type=int, default=1024)
+    ap.add_argument("--out", default="android_agent_results.json")
+    a = ap.parse_args()
+
+    rows, correct_tool, correct_args = [], 0, 0
+    for sc in SCENARIOS:
+        msgs = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content":
+                f"Task: {sc['task']}\n\nCurrent screen:\n{sc['screen']}\n\n"
+                "Choose the next action."},
+        ]
+        try:
+            out, dt = chat(a.base_url, a.model, msgs, TOOLS,
+                           max_tokens=a.max_tokens, think=not a.no_think)
+        except Exception as e:
+            rows.append({"id": sc["id"], "error": str(e)[:200]})
+            print(f"{sc['id']:<20} ERROR {str(e)[:90]}", flush=True)
+            continue
+        msg = out["choices"][0]["message"]
+        calls = msg.get("tool_calls") or []
+        got_name = calls[0]["function"]["name"] if calls else None
+        try:
+            got_args = json.loads(calls[0]["function"]["arguments"]) if calls else {}
+        except Exception:
+            got_args = {"_raw": calls[0]["function"]["arguments"]}
+
+        name_ok = got_name in sc["accept_names"]
+        args_ok = name_ok and all(
+            str(got_args.get(k)) == str(v) for k, v in sc["expect"]["args"].items()
+        ) if got_name == sc["expect"]["name"] else name_ok and not sc["expect"]["args"]
+        correct_tool += int(name_ok)
+        correct_args += int(bool(args_ok))
+        rows.append({
+            "id": sc["id"], "latency_s": round(dt, 2),
+            "expected": sc["expect"], "got_name": got_name, "got_args": got_args,
+            "tool_ok": name_ok, "args_ok": bool(args_ok),
+            "reasoning_chars": len(msg.get("reasoning_content") or ""),
+            "completion_tokens": out.get("usage", {}).get("completion_tokens"),
+        })
+        flag = "OK " if args_ok else ("~  " if name_ok else "X  ")
+        print(f"{flag}{sc['id']:<20} {got_name}({got_args}) {dt:.1f}s", flush=True)
+
+    n = len(SCENARIOS)
+    summary = {
+        "scenarios": n,
+        "thinking": not a.no_think,
+        "tool_accuracy": f"{correct_tool}/{n}",
+        "action_accuracy": f"{correct_args}/{n}",
+        "median_latency_s": round(
+            sorted(r["latency_s"] for r in rows if "latency_s" in r)[
+                max(0, len([r for r in rows if "latency_s" in r]) // 2)] , 2)
+        if any("latency_s" in r for r in rows) else None,
+        "rows": rows,
+    }
+    json.dump(summary, open(a.out, "w"), indent=2)
+    print(json.dumps({k: v for k, v in summary.items() if k != "rows"}, indent=2))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/android_agent/run_episode.py

@@ -0,0 +1,106 @@
+"""Drive Spark-X2.5 through a whole phone task, not one decision.
+
+    # against the in-process simulated phone (no device needed)
+    python run_episode.py --task "Turn on Wi-Fi." --device sim
+
+    # against a real phone
+    adb devices                      # confirm it is listed
+    python run_episode.py --task "Turn on Wi-Fi." --device adb
+
+The server side is vLLM's stock OpenAI endpoint; Spark emits GLM-4.5's tool
+format, so no custom parser is needed:
+
+    vllm serve <model> --port 8077 --enable-auto-tool-choice \\
+        --tool-call-parser glm45 --reasoning-parser glm45
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+
+from agent import AgentConfig, run_episode
+from device import AdbDevice, AdbError, demo_phone
+
+TASKS = {
+    "wifi": "Turn on Wi-Fi.",
+    "alarm": "Set an alarm for 7:30 AM.",
+}
+
+
+def build_device(kind: str, serial: str | None):
+    if kind == "sim":
+        return demo_phone(), "simulated phone (settings + clock)"
+    try:
+        device = AdbDevice(serial=serial)
+        w, h = device.size()
+        return device, f"adb device {serial or '(default)'} {w}x{h}"
+    except AdbError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        raise SystemExit(2)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--task", default=TASKS["wifi"],
+                    help=f"free text, or one of: {', '.join(TASKS)}")
+    ap.add_argument("--device", choices=("sim", "adb"), default="sim")
+    ap.add_argument("--serial", default=None)
+    ap.add_argument("--base-url", default="http://127.0.0.1:8077/v1")
+    ap.add_argument("--model", default="XHToken/Spark-X2.5-1.7B")
+    ap.add_argument("--max-steps", type=int, default=12)
+    ap.add_argument("--max-tokens", type=int, default=1024)
+    ap.add_argument("--no-think", action="store_true")
+    ap.add_argument("--no-think-fallback", action="store_true")
+    ap.add_argument("--out", default=None)
+    a = ap.parse_args()
+
+    task = TASKS.get(a.task, a.task)
+    device, described = build_device(a.device, a.serial)
+    cfg = AgentConfig(
+        base_url=a.base_url, model=a.model, max_steps=a.max_steps,
+        max_tokens=a.max_tokens, think=not a.no_think,
+        think_fallback=not a.no_think_fallback,
+    )
+
+    print(f"device: {described}")
+    print(f"task:   {task}")
+    t0 = time.perf_counter()
+    episode = run_episode(device, task, cfg)
+    wall = time.perf_counter() - t0
+
+    print()
+    print(episode.transcript())
+    latencies = [s.latency_s for s in episode.steps]
+    print(f"\n{episode.n_steps} steps in {wall:.1f} s"
+          f"  (median decision {sorted(latencies)[len(latencies) // 2]:.2f} s)"
+          if latencies else f"\n{episode.n_steps} steps in {wall:.1f} s")
+    if hasattr(device, "state"):
+        print(f"device state: {device.state}")
+
+    if a.out:
+        payload = {
+            "task": task, "device": described, "done": episode.done,
+            "summary": episode.summary, "stop_reason": episode.stop_reason,
+            "wall_s": round(wall, 2),
+            "think": cfg.think, "max_steps": cfg.max_steps,
+            "steps": [
+                {"n": s.n, "tool": s.tool, "args": s.args,
+                 "ok": s.result.ok if s.result else False,
+                 "message": s.result.message if s.result else s.note,
+                 "latency_s": round(s.latency_s, 3), "thinking": s.thinking}
+                for s in episode.steps
+            ],
+            "final_state": getattr(device, "state", None),
+        }
+        with open(a.out, "w") as f:
+            json.dump(payload, f, indent=2, ensure_ascii=False)
+        print(f"wrote {a.out}")
+
+    raise SystemExit(0 if episode.done else 1)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/android_agent/run_suite.py

@@ -0,0 +1,151 @@
+"""Run the task suite and score it, so an agent change can be compared.
+
+Success is decided by the phone's own settings and screen, not by the
+transcript: an agent that says it turned Wi-Fi on and left it off fails here,
+which is exactly the failure this model exhibits.
+
+Two things changed after the plan review:
+
+* ``--repeat N`` now walks the task's **variants** rather than re-running one
+  fixture. At temperature 0 against fixed fixtures, repeats measure
+  repeatability; they only become trials if the phone differs between them.
+  The summary reports a pass rate per task with the variants named, so a build
+  comparison rests on more than one cell.
+* ``--device adb`` scores a real phone. Each run resets the device to the
+  task's declared starting state first, and a task that cannot be scored from
+  a phone (because its predicate needs flip history) is skipped and reported
+  as skipped, rather than silently scored from final state.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import time
+from collections import defaultdict
+
+from agent import AgentConfig, run_episode, wait_for_server
+from phone_state import read_state
+from tasks import SUITE, SUITE_BY_ID
+
+
+def _reset_real_device(device, task) -> None:
+    if not task.reset_adb:
+        raise SystemExit(
+            f"task {task.task_id!r} declares no reset_adb, so a phone run would "
+            f"be scored against an unknown starting state. Add one or exclude "
+            f"the task with --tasks."
+        )
+    for cmd in task.reset_adb:
+        device._run(*cmd)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--base-url", default="http://127.0.0.1:8077/v1")
+    ap.add_argument("--model", default="spark")
+    ap.add_argument("--tasks", default="all",
+                    help="comma-separated task ids, or 'all'")
+    ap.add_argument("--repeat", type=int, default=1,
+                    help="runs per task; each walks the next variant")
+    ap.add_argument("--device", choices=("sim", "adb"), default="sim")
+    ap.add_argument("--serial", default=None, help="adb device serial")
+    ap.add_argument("--no-think", action="store_true")
+    ap.add_argument("--max-tokens", type=int, default=1024)
+    ap.add_argument("--label", default="")
+    ap.add_argument("--out", default=None)
+    a = ap.parse_args()
+
+    tasks = SUITE if a.tasks == "all" else [SUITE_BY_ID[t] for t in a.tasks.split(",")]
+    cfg = AgentConfig(base_url=a.base_url, model=a.model,
+                      think=not a.no_think, max_tokens=a.max_tokens)
+
+    phone = None
+    if a.device == "adb":
+        from device import AdbDevice
+        phone = AdbDevice(serial=a.serial)
+        skipped = [t.task_id for t in tasks if t.needs]
+        if skipped:
+            print(f"skipping on a real phone (needs flip history): "
+                  f"{', '.join(skipped)}")
+        tasks = [t for t in tasks if not t.needs]
+
+    if not wait_for_server(cfg, timeout_s=600):
+        raise SystemExit(f"no model server at {a.base_url}")
+
+    rows, t0 = [], time.perf_counter()
+    for task in tasks:
+        for run in range(a.repeat):
+            if phone is not None:
+                _reset_real_device(phone, task)
+                device = phone
+            else:
+                device = task.build(run)
+            cfg.max_steps = task.max_steps
+            episode = run_episode(device, task.prompt, cfg)
+            score = task.score(read_state(device), episode.done, run)
+            score.update(
+                run=run + 1, steps=episode.n_steps,
+                stop_reason=episode.stop_reason,
+                recovered_steps=sum(1 for s in episode.steps
+                                    if s.parsed_via not in ("tool_calls", "none")),
+                median_latency_s=round(
+                    sorted(s.latency_s for s in episode.steps)[episode.n_steps // 2], 2)
+                if episode.n_steps else None,
+            )
+            rows.append(score)
+            mark = "PASS" if score["success"] else "fail"
+            detail = []
+            if not score.get("scorable", True):
+                mark, detail = "skip", [score.get("reason", "")]
+            else:
+                if score["reached_goal"] and not score["declared_done"]:
+                    detail.append("goal reached but never stopped")
+                if score["declared_done"] and not score["reached_goal"]:
+                    detail.append("claimed done, goal NOT reached")
+                if score["collateral_damage"]:
+                    detail.append("collateral damage")
+            print(f"  {mark}  {task.task_id:18s} {score['variant']:10s} "
+                  f"steps={score['steps']:2d} "
+                  f"{score['stop_reason'][:34]:36s} {'; '.join(detail)}", flush=True)
+
+    wall = time.perf_counter() - t0
+    scored = [r for r in rows if r.get("scorable", True)]
+    n_pass = sum(1 for r in scored if r["success"])
+    reached = sum(1 for r in scored if r["reached_goal"])
+    claimed = sum(1 for r in scored if r["declared_done"] and not r["reached_goal"])
+
+    by_task: dict[str, list[dict]] = defaultdict(list)
+    for r in scored:
+        by_task[r["task_id"]].append(r)
+    print("\npass rate per task")
+    for task_id, rs in by_task.items():
+        passes = sum(1 for r in rs if r["success"])
+        failed = [r["variant"] for r in rs if not r["success"]]
+        print(f"  {task_id:18s} {passes}/{len(rs)}"
+              + (f"   failed on: {', '.join(failed)}" if failed else ""))
+
+    rates = [sum(1 for r in rs if r["success"]) / len(rs) for rs in by_task.values()]
+    spread = (f"{min(rates):.0%}-{max(rates):.0%}" if rates else "-")
+    print(f"\n{n_pass}/{len(scored)} passed in {wall:.0f}s"
+          f"   (goal reached {reached}/{len(scored)}; "
+          f"falsely claimed done {claimed}; per-task rate {spread}"
+          + (f"; median {statistics.median(rates):.0%}" if rates else "") + ")")
+    if len(scored) and len(by_task) and max(len(rs) for rs in by_task.values()) < 3:
+        print("  NOTE: fewer than 3 runs per task -- enough to show a failure "
+              "reproduces, not to rank builds.")
+
+    if a.out:
+        with open(a.out, "w") as f:
+            json.dump({"label": a.label, "base_url": a.base_url,
+                       "model": a.model, "think": cfg.think,
+                       "device": a.device, "repeat": a.repeat,
+                       "passed": n_pass, "total": len(scored),
+                       "skipped": len(rows) - len(scored),
+                       "wall_s": round(wall, 1), "rows": rows}, f, indent=2)
+        print(f"wrote {a.out}")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/android_agent/scenarios.py

@@ -0,0 +1,113 @@
+"""Android screens as a text-only agent sees them, with the correct next action.
+
+Each screen is a uiautomator-style accessibility dump flattened to one line per
+element -- the same shape droidrun's Portal and AndroidWorld's a11y observation
+hand to a text model. `expect` is the action a competent agent must take next;
+`accept_names` allows a defensible alternative where one genuinely exists.
+"""
+
+SCENARIOS = [
+    {
+        "id": "launch_clock",
+        "task": "Set an alarm for 7:30 AM.",
+        "screen": """[0] TextView "Phone" clickable=true
+[1] TextView "Messages" clickable=true
+[2] TextView "Chrome" clickable=true
+[3] TextView "Clock" clickable=true
+[4] TextView "Settings" clickable=true
+[5] TextView "Camera" clickable=true""",
+        "expect": {"name": "tap", "args": {"index": 3}},
+        "accept_names": ["tap", "open_app"],
+    },
+    {
+        "id": "clock_alarm_tab",
+        "task": "Set an alarm for 7:30 AM.",
+        "screen": """[0] Button "Alarm" clickable=true selected=false
+[1] Button "Clock" clickable=true selected=true
+[2] Button "Timer" clickable=true selected=false
+[3] Button "Stopwatch" clickable=true selected=false
+[4] TextView "14:32" clickable=false""",
+        "expect": {"name": "tap", "args": {"index": 0}},
+        "accept_names": ["tap"],
+    },
+    {
+        "id": "add_alarm",
+        "task": "Set an alarm for 7:30 AM.",
+        "screen": """[0] Button "Alarm" clickable=true selected=true
+[1] TextView "No alarms" clickable=false
+[2] ImageButton "Add alarm" clickable=true
+[3] Button "Timer" clickable=true""",
+        "expect": {"name": "tap", "args": {"index": 2}},
+        "accept_names": ["tap"],
+    },
+    {
+        "id": "wifi_settings",
+        "task": "Turn on Wi-Fi.",
+        "screen": """[0] TextView "Settings" clickable=false
+[1] TextView "Network & internet" clickable=true
+[2] TextView "Connected devices" clickable=true
+[3] TextView "Apps" clickable=true
+[4] TextView "Battery" clickable=true
+[5] TextView "Display" clickable=true""",
+        "expect": {"name": "tap", "args": {"index": 1}},
+        "accept_names": ["tap"],
+    },
+    {
+        "id": "wifi_toggle",
+        "task": "Turn on Wi-Fi.",
+        "screen": """[0] TextView "Network & internet" clickable=false
+[1] Switch "Wi-Fi" checked=false clickable=true
+[2] TextView "Mobile network" clickable=true
+[3] Switch "Airplane mode" checked=false clickable=true""",
+        "expect": {"name": "tap", "args": {"index": 1}},
+        "accept_names": ["tap"],
+    },
+    {
+        "id": "search_contact",
+        "task": "Find the contact named Wei Zhang.",
+        "screen": """[0] EditText "Search contacts" clickable=true focused=true
+[1] TextView "Alice Chen" clickable=true
+[2] TextView "Bob Lee" clickable=true
+[3] TextView "Carol Wu" clickable=true""",
+        "expect": {"name": "type_text", "args": {"index": 0, "text": "Wei Zhang"}},
+        "accept_names": ["type_text"],
+    },
+    {
+        "id": "scroll_for_item",
+        "task": "Open the Developer options screen in Settings.",
+        "screen": """[0] TextView "Settings" clickable=false
+[1] TextView "Display" clickable=true
+[2] TextView "Sound" clickable=true
+[3] TextView "Storage" clickable=true
+[4] TextView "Privacy" clickable=true
+[5] TextView "Location" clickable=true""",
+        "expect": {"name": "swipe", "args": {"direction": "up"}},
+        "accept_names": ["swipe"],
+    },
+    {
+        "id": "go_back",
+        "task": "Return to the previous screen; you opened the wrong menu.",
+        "screen": """[0] TextView "About phone" clickable=false
+[1] TextView "Model: Pixel 8" clickable=false
+[2] TextView "Android version" clickable=true""",
+        "expect": {"name": "press_key", "args": {"key": "back"}},
+        "accept_names": ["press_key"],
+    },
+    {
+        "id": "open_app_by_name",
+        "task": "Open the Camera app.",
+        "screen": """[0] TextView "Swipe up to unlock" clickable=false
+[1] TextView "10:04" clickable=false""",
+        "expect": {"name": "open_app", "args": {"name": "Camera"}},
+        "accept_names": ["open_app", "swipe"],
+    },
+    {
+        "id": "task_done",
+        "task": "Turn on Wi-Fi.",
+        "screen": """[0] TextView "Network & internet" clickable=false
+[1] Switch "Wi-Fi" checked=true clickable=true
+[2] TextView "Connected to HOME-5G" clickable=false""",
+        "expect": {"name": "task_complete", "args": {}},
+        "accept_names": ["task_complete"],
+    },
+]

benchmarks/edge_harness/android_agent/tasks.py

@@ -0,0 +1,279 @@
+"""A scored task suite, judged by what the phone shows rather than by what the
+agent said -- or by what the simulator happens to remember.
+
+Three things were wrong with the first version of this file, and all three
+made it score more generously than it claimed to:
+
+* **``add_alarm`` passed on any typed string containing "7".** Typing ``7``
+  and stopping scored a success on a task whose prompt is "Set an alarm for
+  7:30 AM": no minute, no meridiem, and no requirement that the alarm was ever
+  saved. It now requires a *committed* alarm reading 7:30 AM, which the clock
+  screen lists, so the check is one a real phone can also answer.
+* **``wifi_on_already`` scored final state only.** Its note said "any tap on
+  the switch fails the task"; the code could not see a tap, so turning Wi-Fi
+  off and back on passed. Switch flips are now recorded and the task fails on
+  any of them -- which is the exact failure W4A8 exhibited (it turned Wi-Fi on
+  and then tapped the switch off again).
+* **Predicates read ``SimulatedDevice`` attributes**, so every scored run was a
+  simulator run by construction. They read :class:`PhoneState` now, which comes
+  from the UI dump for both backends.
+
+Repeats are only trials if something differs between them, so each task
+carries **variants**: the same goal against a renamed control, a reordered
+screen, or a different starting state. Five runs of one fixture at temperature
+0 measure repeatability; five variants measure whether the agent can do the
+task.
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass, field
+from typing import Callable
+
+from device import SimElement, SimulatedDevice, demo_phone
+from phone_state import PhoneState, read_state
+
+Predicate = Callable[[PhoneState], bool]
+
+CLOCK_PACKAGE = "com.google.android.deskclock"
+"""Clock app to reset for the alarm tasks. AOSP builds and several OEMs ship a
+different package (``com.android.deskclock``, ``com.sec.android.app.clockpackage``
+on Samsung); override before a real-device run, because clearing the wrong
+package leaves a stale alarm and the next run scores against it."""
+
+
+# --------------------------------------------------------------- variations
+
+def rename(screen: str, old: str, new: str) -> Callable[[SimulatedDevice], None]:
+    """Same control, different word. Android OEMs really do ship "WLAN"."""
+    def apply(d: SimulatedDevice) -> None:
+        for el in d.screens[screen].elements:
+            if el.label == old:
+                el.label = new
+    return apply
+
+
+def reorder(screen: str, seed: int) -> Callable[[SimulatedDevice], None]:
+    """Shuffle the tappable rows. Index-based tapping must not be luck."""
+    def apply(d: SimulatedDevice) -> None:
+        els = d.screens[screen].elements
+        head = [e for e in els if not e.clickable]
+        body = [e for e in els if e.clickable]
+        random.Random(seed).shuffle(body)
+        d.screens[screen].elements = head + body
+    return apply
+
+
+def decoy(screen: str, label: str) -> Callable[[SimulatedDevice], None]:
+    """A plausible wrong row, e.g. "Wi-Fi calling" next to "Wi-Fi"."""
+    def apply(d: SimulatedDevice) -> None:
+        d.screens[screen].elements.append(SimElement(label))
+    return apply
+
+
+def chain(*fns: Callable[[SimulatedDevice], None]) -> Callable[[SimulatedDevice], None]:
+    def apply(d: SimulatedDevice) -> None:
+        for fn in fns:
+            fn(d)
+    return apply
+
+
+@dataclass
+class Variant:
+    variant_id: str
+    mutate: Callable[[SimulatedDevice], None] = lambda d: None
+
+
+# --------------------------------------------------------------------- task
+
+@dataclass
+class Task:
+    task_id: str
+    prompt: str
+    reaches: Predicate
+    must_not: Predicate = lambda s: False
+    setup: Callable[[SimulatedDevice], None] = lambda d: None
+    max_steps: int = 10
+    note: str = ""
+    variants: list[Variant] = field(default_factory=lambda: [Variant("base")])
+    needs: tuple[str, ...] = ()
+    """PhoneState facts this task's predicates require that a screen cannot
+    show (``toggled``, ``fields``, ``records``). A task listing any of these
+    cannot be scored on a real phone, and says so instead of scoring wrong."""
+    reset_adb: tuple[tuple[str, ...], ...] = ()
+    """Commands that put a real phone into this task's starting state. Without
+    them a real-device run is scored against an unknown start, so an empty
+    tuple means "refuse to run me on a phone"."""
+
+    def variant(self, n: int) -> Variant:
+        return self.variants[n % len(self.variants)]
+
+    def build(self, run: int = 0) -> SimulatedDevice:
+        device = demo_phone()
+        self.variant(run).mutate(device)
+        self.setup(device)
+        return device
+
+    def scorable_on(self, state: PhoneState) -> str | None:
+        """Why this task cannot be scored from ``state``, or None if it can."""
+        if state.source == "device" and self.needs:
+            return (f"needs {', '.join(self.needs)}, which a screen cannot "
+                    f"show; simulator only")
+        return None
+
+    def score(self, device_or_state, declared_done: bool, run: int = 0) -> dict:
+        state = (device_or_state if isinstance(device_or_state, PhoneState)
+                 else read_state(device_or_state))
+        blocked = self.scorable_on(state)
+        if blocked:
+            return {"task_id": self.task_id, "variant": self.variant(run).variant_id,
+                    "scorable": False, "reason": blocked, "success": False,
+                    "reached_goal": None, "collateral_damage": None,
+                    "declared_done": declared_done}
+        reached = bool(self.reaches(state))
+        damaged = bool(self.must_not(state))
+        return {
+            "task_id": self.task_id,
+            "variant": self.variant(run).variant_id,
+            "scorable": True,
+            "reached_goal": reached,
+            "collateral_damage": damaged,
+            "declared_done": declared_done,
+            # The only row that counts: the phone is in the requested state,
+            # nothing else was broken, and the agent knew to stop.
+            "success": reached and not damaged and declared_done,
+        }
+
+
+# ----------------------------------------------------------------- oracles
+
+# Both the on-screen labels and the setting key, because `switch()` matches
+# either source: "Wi-Fi" normalises onto the `wifi` setting by luck, but
+# "Airplane mode" does not normalise onto `airplane`, and a name the setting
+# lookup misses silently falls through to "control not on screen".
+WIFI = ("Wi-Fi", "WiFi", "WLAN", "wifi")
+AIRPLANE = ("Airplane mode", "Flight mode", "airplane")
+
+
+def _wifi_is(want: bool) -> Predicate:
+    # `switch` returns None when the control is not on screen, which is not
+    # the same as off: an agent that never navigated there has not turned
+    # anything on.
+    return lambda s: s.switch(*WIFI) is want
+
+
+def _airplane_on(s: PhoneState) -> bool:
+    return s.switch(*AIRPLANE) is True
+
+
+def _alarm_set(hour: int, minute: int, pm: bool) -> Predicate:
+    """A *saved* alarm reading the requested time.
+
+    The clock screen lists committed alarms, so this reads the screen and works
+    on a real phone. Typing the digits without saving leaves the list empty.
+    """
+    wanted = f"{hour}:{minute:02d} {'PM' if pm else 'AM'}"
+    return lambda s: s.has_text(wanted)
+
+
+SUITE: list[Task] = [
+    Task(
+        task_id="wifi_on",
+        prompt="Turn on Wi-Fi.",
+        reaches=_wifi_is(True),
+        must_not=_airplane_on,
+        note="Two screens deep, then a switch. The switch's state is visible, "
+             "so stopping is decidable from the screen.",
+        variants=[
+            Variant("base"),
+            Variant("wlan", rename("network", "Wi-Fi", "WLAN")),
+            Variant("reordered", reorder("network", seed=7)),
+            Variant("decoy", decoy("network", "Wi-Fi calling")),
+            Variant("deep", chain(reorder("settings", seed=3),
+                                  decoy("network", "Wi-Fi preferences"))),
+        ],
+        reset_adb=(("shell", "svc", "wifi", "disable"),),
+    ),
+    Task(
+        task_id="wifi_on_already",
+        prompt="Turn on Wi-Fi.",
+        setup=lambda d: d.state.__setitem__("wifi", True),
+        reaches=lambda s: s.switch(*WIFI) is True and s.times_toggled("wifi") == 0,
+        must_not=_airplane_on,
+        note="Already satisfied. The only correct action is task_complete. "
+             "Any flip of the switch fails, including off-and-on again, which "
+             "final state cannot see and which W4A8 actually did.",
+        needs=("toggled",),
+        variants=[
+            Variant("base"),
+            Variant("wlan", rename("network", "Wi-Fi", "WLAN")),
+            Variant("reordered", reorder("network", seed=11)),
+        ],
+    ),
+    Task(
+        task_id="wifi_off",
+        prompt="Turn off Wi-Fi.",
+        setup=lambda d: d.state.__setitem__("wifi", True),
+        reaches=_wifi_is(False),
+        must_not=_airplane_on,
+        note="The mirror of wifi_on: guards against a model that has simply "
+             "learned to tap switches.",
+        variants=[
+            Variant("base"),
+            Variant("wlan", rename("network", "Wi-Fi", "WLAN")),
+            Variant("reordered", reorder("network", seed=5)),
+        ],
+        reset_adb=(("shell", "svc", "wifi", "enable"),),
+    ),
+    Task(
+        task_id="airplane_on",
+        prompt="Turn on airplane mode.",
+        reaches=_airplane_on,
+        note="Same screen as Wi-Fi, different row -- catches index confusion, "
+             "which is why one variant reorders that screen.",
+        variants=[
+            Variant("base"),
+            Variant("reordered", reorder("network", seed=2)),
+            Variant("decoy", decoy("network", "Airplane mode settings")),
+        ],
+        # `settings put global` needs WRITE_SECURE_SETTINGS, which adb has on a
+        # debuggable build but not on every retail phone; the run fails loudly
+        # if it is refused rather than scoring against an unknown start.
+        reset_adb=(("shell", "settings", "put", "global", "airplane_mode_on", "0"),),
+    ),
+    Task(
+        task_id="open_clock",
+        prompt="Open the Clock app and go to the Alarm tab.",
+        reaches=lambda s: s.has_text("Add alarm") or s.has_text("Set time"),
+        note="Navigation only, no state change. Isolates 'can it find an app' "
+             "from 'can it finish'.",
+        variants=[
+            Variant("base"),
+            Variant("reordered", reorder("home", seed=4)),
+            Variant("decoy", decoy("home", "Clock Widget")),
+        ],
+        reset_adb=(("shell", "am", "force-stop", CLOCK_PACKAGE),
+                   ("shell", "input", "keyevent", "KEYCODE_HOME")),
+    ),
+    Task(
+        task_id="add_alarm",
+        prompt="Set an alarm for 7:30 AM.",
+        reaches=_alarm_set(7, 30, pm=False),
+        must_not=lambda s: s.has_text("7:30 PM"),
+        max_steps=12,
+        note="The full flow: right app, right tab, add, type both fields, "
+             "leave the meridiem alone, save. Previously this passed on any "
+             "typed string containing '7'.",
+        variants=[
+            Variant("base"),
+            Variant("reordered", reorder("clock", seed=9)),
+            Variant("decoy", decoy("alarms", "Alarm settings")),
+        ],
+        # `pm clear` removes saved alarms, so run N does not see run N-1's.
+        reset_adb=(("shell", "pm", "clear", CLOCK_PACKAGE),
+                   ("shell", "input", "keyevent", "KEYCODE_HOME")),
+    ),
+]
+
+SUITE_BY_ID = {t.task_id: t for t in SUITE}

benchmarks/edge_harness/android_agent/test_android_agent.py

@@ -0,0 +1,525 @@
+"""Tests for the Android agent toolkit.
+
+No phone, emulator or model server is needed: the device is the in-process
+simulator and the model is a scripted stand-in. That is deliberate -- this
+host has no `adb`, no emulator and no `/dev/kvm`, so anything that could only
+be checked against real hardware would otherwise ship unverified.
+"""
+
+import json
+
+import pytest
+
+import agent as agent_mod
+from agent import AgentConfig, first_tool_call, run_episode
+from device import demo_phone
+from executor import execute
+from ui_tree import parse_ui_dump, render_tree, screen_signature
+
+# --------------------------------------------------------------------- ui_tree
+
+REAL_DUMP = """<?xml version="1.0" encoding="UTF-8"?>
+<hierarchy rotation="0">
+ <node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.android.settings" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1080,2400]">
+  <node index="1" text="Network &amp; internet" resource-id="android:id/title" class="android.widget.TextView" package="com.android.settings" content-desc="" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,300][1080,460]" />
+  <node index="2" text="" resource-id="" class="android.widget.LinearLayout" package="com.android.settings" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,460][1080,620]" />
+  <node index="3" text="" resource-id="com.android.settings:id/switch_widget" class="android.widget.Switch" package="com.android.settings" content-desc="Wi-Fi" checkable="true" checked="true" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[800,460][1000,620]" />
+ </node>
+</hierarchy>"""
+
+
+def test_parses_a_real_dump_and_keeps_only_what_is_actionable():
+    els = parse_ui_dump(REAL_DUMP)
+    labels = [e.text for e in els]
+    assert "Network & internet" in labels          # entity decoded
+    assert "Wi-Fi" in labels                        # fell back to content-desc
+    assert "" not in labels                         # bare layout node dropped
+    assert [e.index for e in els] == list(range(len(els)))
+
+
+def test_switch_state_is_visible_to_the_model():
+    """The one clearly wrong action measured earlier was tapping a switch that
+    was already on; that is only avoidable if the state is on the screen."""
+    rendered = render_tree(parse_ui_dump(REAL_DUMP))
+    assert "checked=true" in rendered
+    assert 'Switch "Wi-Fi"' in rendered
+
+
+def test_index_maps_to_the_centre_of_the_element():
+    els = parse_ui_dump(REAL_DUMP)
+    wifi = next(e for e in els if e.text == "Wi-Fi")
+    assert wifi.center == (900, 540)
+
+
+def test_signature_tracks_state_not_pixels():
+    a = parse_ui_dump(REAL_DUMP)
+    shifted = REAL_DUMP.replace("[0,300][1080,460]", "[0,310][1080,470]")
+    assert screen_signature(parse_ui_dump(shifted)) == screen_signature(a)
+    toggled = REAL_DUMP.replace('checkable="true" checked="true"',
+                                'checkable="true" checked="false"')
+    assert screen_signature(parse_ui_dump(toggled)) != screen_signature(a)
+
+
+def test_element_cap_bounds_the_context():
+    big = '<?xml version="1.0"?><hierarchy>' + "".join(
+        f'<node text="row{i}" class="android.widget.TextView" clickable="true" '
+        f'bounds="[0,{i},100,{i + 10}]" />' for i in range(200)
+    ) + "</hierarchy>"
+    assert len(parse_ui_dump(big, max_elements=25)) == 25
+
+
+# -------------------------------------------------------------------- executor
+
+def test_tap_lands_on_the_right_row_and_reports_prior_state():
+    device = demo_phone()
+    device.current = "network"
+    els = parse_ui_dump(device.dump())
+    wifi = next(e for e in els if e.text == "Wi-Fi")
+    result = execute("tap", {"index": wifi.index}, els, device)
+    assert result.ok and device.state["wifi"] is True
+    assert "was off" in result.message
+
+
+def test_out_of_range_index_is_a_message_not_an_exception():
+    device = demo_phone()
+    els = parse_ui_dump(device.dump())
+    result = execute("tap", {"index": 99}, els, device)
+    assert not result.ok
+    assert "out of range" in result.message and f"0-{len(els) - 1}" in result.message
+
+
+def test_bad_arguments_explain_themselves():
+    device = demo_phone()
+    els = parse_ui_dump(device.dump())
+    assert "missing required argument" in execute("tap", {}, els, device).message
+    assert "whole number" in execute("tap", {"index": "banana"}, els, device).message
+    assert "unknown tool" in execute("frobnicate", {}, els, device).message
+    assert "direction must be" in execute("swipe", {"direction": "sideways"}, els, device).message
+
+
+def test_type_text_focuses_the_field_first():
+    device = demo_phone()
+    device.current = "new_alarm"
+    els = parse_ui_dump(device.dump())
+    field = next(e for e in els if e.text == "Hour")
+    execute("type_text", {"index": field.index, "text": "07"}, els, device)
+    kinds = [a[0] for a in device.actions]
+    assert kinds == ["tap", "text"], kinds
+    assert device.typed == ["07"]
+
+
+def test_swipe_direction_is_content_movement():
+    device = demo_phone()
+    els = parse_ui_dump(device.dump())
+    execute("swipe", {"direction": "down"}, els, device)
+    _, x1, y1, x2, y2 = device.actions[-1]
+    assert int(y1) > int(y2), "scrolling down must move the finger up"
+
+
+def test_task_complete_carries_its_summary():
+    device = demo_phone()
+    result = execute("task_complete", {"summary": "Wi-Fi is on"}, [], device)
+    assert result.done and result.summary == "Wi-Fi is on"
+
+
+# ----------------------------------------------------------------------- agent
+
+def _scripted(calls):
+    """A stand-in model that plays a fixed list of tool calls."""
+    seq = list(calls)
+
+    def fake_chat(cfg, messages, *, think):
+        call = seq.pop(0) if seq else None
+        if call is None:
+            return {"choices": [{"message": {"content": "", "tool_calls": []}}]}, 0.01
+        name, args = call
+        return {"choices": [{"message": {"tool_calls": [
+            {"function": {"name": name, "arguments": json.dumps(args)}}
+        ]}}]}, 0.01
+
+    return fake_chat
+
+
+def test_first_tool_call_handles_both_argument_encodings():
+    as_str = {"choices": [{"message": {"tool_calls": [
+        {"function": {"name": "tap", "arguments": '{"index": 2}'}}]}}]}
+    as_dict = {"choices": [{"message": {"tool_calls": [
+        {"function": {"name": "tap", "arguments": {"index": 2}}}]}}]}
+    assert first_tool_call(as_str) == ("tap", {"index": 2})
+    assert first_tool_call(as_dict) == ("tap", {"index": 2})
+    assert first_tool_call({"choices": [{"message": {"content": "hi"}}]}) == (None, {})
+    # Malformed JSON must not kill the episode.
+    broken = {"choices": [{"message": {"tool_calls": [
+        {"function": {"name": "tap", "arguments": "{not json"}}]}}]}
+    assert first_tool_call(broken) == ("tap", {})
+
+
+def test_completes_a_multi_screen_task(monkeypatch):
+    device = demo_phone()
+    monkeypatch.setattr(agent_mod, "chat", _scripted([
+        ("tap", {"index": 3}),                       # home -> Settings
+        ("tap", {"index": 1}),                       # -> Network & internet
+        ("tap", {"index": 1}),                       # Wi-Fi switch
+        ("task_complete", {"summary": "Wi-Fi enabled"}),
+    ]))
+    ep = run_episode(device, "Turn on Wi-Fi.", AgentConfig(max_steps=8))
+    assert ep.done and device.state["wifi"] is True
+    assert ep.summary == "Wi-Fi enabled"
+    assert ep.stop_reason == "task_complete"
+    assert ep.n_steps == 4
+
+
+def test_says_so_when_an_action_changes_nothing(monkeypatch):
+    device = demo_phone()
+    monkeypatch.setattr(agent_mod, "chat", _scripted([
+        ("tap", {"index": 0}),  # "Phone" on home: goes nowhere
+        ("task_complete", {"summary": "done"}),
+    ]))
+    ep = run_episode(device, "Do something.", AgentConfig(max_steps=4))
+    assert "did not change" in ep.steps[0].result.message
+
+
+def test_stops_when_it_repeats_a_no_op(monkeypatch):
+    device = demo_phone()
+    monkeypatch.setattr(agent_mod, "chat", _scripted([("tap", {"index": 0})] * 6))
+    ep = run_episode(device, "Do something.", AgentConfig(max_steps=6))
+    assert not ep.done
+    assert "stuck" in ep.stop_reason
+    assert ep.n_steps < 6, "should give up before the budget"
+
+
+def test_empty_reply_retries_without_thinking(monkeypatch):
+    """The measured failure: reasoning never closes `<think>`, so the parser
+    returns nothing. A non-thinking retry is worth ~0.6 s."""
+    seen = []
+
+    def fake_chat(cfg, messages, *, think):
+        seen.append(think)
+        if think:
+            return {"choices": [{"message": {"content": "", "tool_calls": []}}]}, 8.9
+        return {"choices": [{"message": {"tool_calls": [
+            {"function": {"name": "task_complete",
+                          "arguments": '{"summary": "fallback"}'}}]}}]}, 0.61
+
+    monkeypatch.setattr(agent_mod, "chat", fake_chat)
+    ep = run_episode(device := demo_phone(), "Task.", AgentConfig(max_steps=3))
+    assert seen == [True, False], seen
+    assert ep.done and ep.summary == "fallback"
+    assert ep.steps[0].thinking is False
+    assert device is not None
+
+
+def test_gives_up_rather_than_looping_when_no_tool_call_ever_comes(monkeypatch):
+    monkeypatch.setattr(agent_mod, "chat", _scripted([]))
+    ep = run_episode(demo_phone(), "Task.", AgentConfig(max_steps=5))
+    assert not ep.done
+    assert "no tool call" in ep.stop_reason
+    assert ep.n_steps == 1
+
+
+def test_step_budget_is_enforced(monkeypatch):
+    device = demo_phone()
+    # Alternate two real navigations so nothing is a repeat or a no-op.
+    monkeypatch.setattr(agent_mod, "chat", _scripted(
+        [("tap", {"index": 3}), ("press_key", {"key": "back"})] * 5))
+    ep = run_episode(device, "Wander.", AgentConfig(max_steps=4))
+    assert not ep.done
+    assert ep.n_steps == 4
+    assert "budget exhausted" in ep.stop_reason
+
+
+def test_transcript_is_readable(monkeypatch):
+    monkeypatch.setattr(agent_mod, "chat", _scripted([
+        ("tap", {"index": 3}), ("task_complete", {"summary": "ok"})]))
+    ep = run_episode(demo_phone(), "Open settings.", AgentConfig(max_steps=4))
+    text = ep.transcript()
+    assert "task: Open settings." in text
+    assert "tap(" in text and "task_complete" in text
+    assert "stopped: task_complete" in text
+
+
+def test_detects_undoing_its_own_action(monkeypatch):
+    """The measured failure: tapping a switch that is already on, turning it
+    back off, forever. Each tap really does change the screen, so the
+    no-change check cannot see it."""
+    device = demo_phone()
+    device.current = "network"
+    monkeypatch.setattr(agent_mod, "chat", _scripted([("tap", {"index": 1})] * 6))
+    ep = run_episode(device, "Turn on Wi-Fi.", AgentConfig(max_steps=6))
+    assert not ep.done
+    assert "undoing its own action" in ep.stop_reason
+    assert any("undid your previous action" in s.result.message
+               for s in ep.steps if s.result)
+
+
+def test_normal_navigation_is_not_mistaken_for_oscillation(monkeypatch):
+    """Opening a submenu and pressing back revisits a screen; that is how
+    phones work and must not trip the guard."""
+    device = demo_phone()
+    monkeypatch.setattr(agent_mod, "chat", _scripted([
+        ("tap", {"index": 3}),          # home -> settings
+        ("press_key", {"key": "back"}),  # -> home
+        ("tap", {"index": 2}),          # home -> clock
+        ("press_key", {"key": "back"}),  # -> home
+        ("task_complete", {"summary": "looked around"}),
+    ]))
+    ep = run_episode(device, "Look around.", AgentConfig(max_steps=8))
+    assert ep.done, ep.stop_reason
+    assert ep.n_steps == 5
+
+
+# ------------------------------------------------------------------ task suite
+
+def test_suite_scores_on_device_state_not_on_what_the_agent_said():
+    """The failure this model exhibits is claiming completion it did not
+    achieve, so the score must not read the transcript."""
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on"]
+    device = task.build()
+    assert task.score(device, declared_done=True)["success"] is False
+    device.state["wifi"] = True
+    assert task.score(device, declared_done=True)["success"] is True
+    # Reaching the goal without stopping is not success either.
+    assert task.score(device, declared_done=False)["success"] is False
+
+
+def test_collateral_damage_fails_the_task():
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on"]
+    device = task.build()
+    device.state.update(wifi=True, airplane=True)
+    score = task.score(device, declared_done=True)
+    assert score["reached_goal"] and score["collateral_damage"]
+    assert score["success"] is False
+
+
+def test_already_satisfied_task_starts_satisfied():
+    """Its only correct action is task_complete; tapping the switch fails."""
+    from phone_state import read_state
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on_already"]
+    device = task.build()
+    assert task.reaches(read_state(device))
+    assert task.score(device, declared_done=True)["success"] is True
+
+
+def test_already_satisfied_task_fails_on_a_switch_round_trip():
+    """W4A8 turned Wi-Fi on and then tapped the switch off again. Final state
+    cannot see that, so this task scored it a pass until the flips were
+    recorded."""
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on_already"]
+    device = task.build()
+    device.launch("Settings")
+    device.tap(540, 360)          # Network & internet
+    device.tap(540, 360)          # Wi-Fi off
+    device.tap(540, 360)          # ... and on again
+    assert device.state["wifi"] is True          # final state looks correct
+    assert task.score(device, declared_done=True)["success"] is False
+
+
+def test_goal_is_read_from_the_setting_not_from_the_final_screen():
+    """An agent that turns Wi-Fi on and then goes Home has still done it."""
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on"]
+    device = task.build()
+    device.launch("Settings")
+    device.tap(540, 360)
+    device.tap(540, 360)
+    device.keyevent("home")
+    assert task.score(device, declared_done=True)["success"] is True
+
+
+def test_never_finding_the_screen_is_not_scored_as_off():
+    """`switch()` returns None when neither the setting nor the screen knows
+    the control; a predicate must not read that as 'off' and pass a task."""
+    from phone_state import PhoneState
+    from tasks import SUITE_BY_ID
+
+    empty = PhoneState(source="device")
+    assert empty.switch("Wi-Fi") is None
+    assert SUITE_BY_ID["wifi_off"].score(empty, declared_done=True)["success"] is False
+
+
+def test_suite_includes_a_mirror_task_for_switch_tapping():
+    """Guards against a model that has simply learned 'tap the switch'."""
+    from tasks import SUITE_BY_ID
+
+    from phone_state import read_state
+
+    off = SUITE_BY_ID["wifi_off"]
+    device = off.build()
+    assert device.state["wifi"] is True
+    assert not off.reaches(read_state(device))
+    device.state["wifi"] = False
+    assert off.reaches(read_state(device))
+
+
+def test_chat_retries_transient_failures(monkeypatch):
+    """A phone task is a dozen round trips; one dropped connection should cost
+    a retry, not the episode."""
+    import agent as mod
+    from urllib import error as urlerror
+
+    calls = {"n": 0}
+
+    def flaky(req, timeout=None):
+        calls["n"] += 1
+        if calls["n"] < 3:
+            raise urlerror.URLError("connection reset")
+        class _R:
+            def __enter__(self): return self
+            def __exit__(self, *a): return False
+            def read(self): return b'{"choices":[{"message":{"tool_calls":[]}}]}'
+        return _R()
+
+    monkeypatch.setattr(mod.urlrequest, "urlopen", flaky)
+    monkeypatch.setattr(mod.time, "sleep", lambda s: None)
+    cfg = AgentConfig(retries=3)
+    body, _ = mod.chat(cfg, [{"role": "user", "content": "x"}], think=False)
+    assert calls["n"] == 3 and "choices" in body
+
+
+def test_chat_gives_up_with_a_clear_error(monkeypatch):
+    import agent as mod
+    from urllib import error as urlerror
+
+    monkeypatch.setattr(mod.urlrequest, "urlopen",
+                        lambda *a, **k: (_ for _ in ()).throw(urlerror.URLError("down")))
+    monkeypatch.setattr(mod.time, "sleep", lambda s: None)
+    with pytest.raises(mod.ChatError, match="after 2 attempts"):
+        mod.chat(AgentConfig(retries=2), [{"role": "user", "content": "x"}], think=False)
+
+
+# --------------------------------------------------- oracles that were wrong
+
+def test_alarm_task_needs_a_saved_alarm_not_a_typed_digit():
+    """This oracle used to pass on any typed string containing "7", so typing
+    `7` and stopping scored a success on "Set an alarm for 7:30 AM"."""
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["add_alarm"]
+
+    typed_only = task.build()
+    typed_only.launch("Clock")
+    typed_only.tap(540, 200)      # Alarm tab
+    typed_only.tap(540, 360)      # Add alarm
+    typed_only.tap(540, 360)
+    typed_only.input_text("7")
+    assert task.score(typed_only, declared_done=True)["success"] is False
+
+    both_fields = task.build()
+    both_fields.launch("Clock")
+    both_fields.tap(540, 200)
+    both_fields.tap(540, 360)
+    both_fields.tap(540, 360)
+    both_fields.input_text("7")
+    both_fields.tap(540, 520)
+    both_fields.input_text("30")
+    assert task.score(both_fields, declared_done=True)["success"] is False, \
+        "typing both fields without saving is not a set alarm"
+
+    both_fields.tap(540, 840)     # Save
+    assert task.score(both_fields, declared_done=True)["success"] is True
+
+
+def test_alarm_task_rejects_the_wrong_meridiem():
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["add_alarm"]
+    d = task.build()
+    d.launch("Clock"); d.tap(540, 200); d.tap(540, 360)
+    d.tap(540, 360); d.input_text("7")
+    d.tap(540, 520); d.input_text("30")
+    d.tap(540, 680)               # AM -> PM
+    d.tap(540, 840)               # Save
+    score = task.score(d, declared_done=True)
+    assert score["success"] is False and score["collateral_damage"] is True
+
+
+def test_a_saved_alarm_is_visible_on_the_screen():
+    """The oracle reads the clock app's list, so it works on a real phone."""
+    from phone_state import read_state
+    from tasks import SUITE_BY_ID
+
+    d = SUITE_BY_ID["add_alarm"].build()
+    d.launch("Clock"); d.tap(540, 200); d.tap(540, 360)
+    d.tap(540, 360); d.input_text("7")
+    d.tap(540, 520); d.input_text("30")
+    d.tap(540, 840)
+    assert read_state(d).has_text("7:30 AM")
+
+
+# ------------------------------------------------------------------ variants
+
+def test_every_task_has_variants_so_repeats_are_trials():
+    """Five runs of one fixture at temperature 0 measure repeatability. The
+    suite has to vary something for a repeat to be a trial."""
+    from tasks import SUITE
+
+    for task in SUITE:
+        assert len(task.variants) >= 1
+        ids = [v.variant_id for v in task.variants]
+        assert len(ids) == len(set(ids)), task.task_id
+    assert any(len(t.variants) > 1 for t in SUITE)
+
+
+def test_variants_change_the_phone_but_not_the_goal():
+    """A renamed or reordered control is still the same task."""
+    from phone_state import read_state
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on"]
+    labels_seen = set()
+    for run in range(len(task.variants)):
+        d = task.build(run)
+        d.launch("Settings")
+        d.tap(540, 360)
+        labels_seen.add(read_state(d).source)
+        # The goal predicate is unchanged; only the route differs.
+        assert task.score(d, declared_done=True)["success"] is False
+        d.state["wifi"] = True
+        assert task.score(d, declared_done=True)["success"] is True
+    assert labels_seen == {"sim"}
+
+
+def test_wlan_variant_renames_the_control_the_agent_must_find():
+    from phone_state import read_state
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on"]
+    variant = next(i for i, v in enumerate(task.variants) if v.variant_id == "wlan")
+    d = task.build(variant)
+    d.launch("Settings"); d.tap(540, 360)
+    assert read_state(d).has_text("WLAN")
+    assert not read_state(d).has_text("Wi-Fi")
+
+
+# -------------------------------------------------- real device honesty
+
+def test_history_dependent_task_refuses_to_score_on_a_real_phone():
+    """A phone cannot report which switches were flipped, so the task says so
+    instead of scoring from final state and calling a round trip a pass."""
+    from phone_state import PhoneState
+    from tasks import SUITE_BY_ID
+
+    state = PhoneState(source="device", settings={"wifi": True})
+    score = SUITE_BY_ID["wifi_on_already"].score(state, declared_done=True)
+    assert score["scorable"] is False
+    assert "simulator only" in score["reason"]
+    assert score["success"] is False
+
+
+def test_tasks_runnable_on_a_phone_declare_how_to_reset_it():
+    """Without a reset the run is scored against an unknown starting state."""
+    from tasks import SUITE_BY_ID
+
+    assert SUITE_BY_ID["wifi_on"].reset_adb
+    assert SUITE_BY_ID["wifi_off"].reset_adb
+    # and the sim-only one is not pretending otherwise
+    assert SUITE_BY_ID["wifi_on_already"].needs == ("toggled",)

benchmarks/edge_harness/android_agent/tool_parse.py

@@ -0,0 +1,155 @@
+"""Recover a tool call from what the model actually emitted.
+
+vLLM's stock `glm45` parser handles Spark's tool output most of the time --
+4 of 5 screens in a spot check -- and then does not. On the Wi-Fi toggle
+screen, the one where a switch's state decides the right action, the model
+dropped out of GLM-4.5 format entirely and emitted
+
+    [tap]
+    {"tool": "tap", "parameters": {"index": 1}
+
+with the JSON unterminated. `tool_calls` comes back empty, and an agent that
+only reads `tool_calls` sees a step with no action at all.
+
+For a phone agent that is the difference between finishing a task and
+stopping in the middle of it, so this module tries, in order:
+
+1. the parsed `tool_calls` (the normal path -- nothing here second-guesses it),
+2. a GLM-4.5 `<tool_call>` block left in the content,
+3. the bracketed `[name]` + JSON form above, repairing unbalanced braces,
+4. a bare JSON object naming a tool.
+
+Every recovery is reported, because an agent that quietly repairs its model's
+output will hide a regression in that model.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+
+_BRACKET = re.compile(r"\[([a-z_][a-z0-9_]*)\]", re.I)
+_GLM_CALL = re.compile(
+    r"<tool_call>\s*([a-z_][a-z0-9_]*)(.*?)</tool_call>", re.I | re.S)
+_GLM_ARG = re.compile(
+    r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>", re.I | re.S)
+
+_ARG_CONTAINERS = ("parameters", "arguments", "args")
+_NAME_KEYS = ("tool", "name", "tool_name", "function")
+
+
+@dataclass
+class ParsedCall:
+    name: str | None
+    args: dict
+    source: str
+    """Which path produced it: tool_calls | glm_block | bracketed | bare_json."""
+
+    @property
+    def recovered(self) -> bool:
+        return self.name is not None and self.source != "tool_calls"
+
+
+def _coerce(value: str):
+    """Tool arguments arrive as strings; indices must become integers."""
+    text = value.strip()
+    if re.fullmatch(r"-?\d+", text):
+        return int(text)
+    if text.lower() in ("true", "false"):
+        return text.lower() == "true"
+    return text
+
+
+def _balance(text: str) -> str:
+    """Close a JSON object the model stopped short of finishing."""
+    depth_curly = text.count("{") - text.count("}")
+    depth_square = text.count("[") - text.count("]")
+    if depth_curly < 0 or depth_square < 0:
+        return text
+    return text + "]" * depth_square + "}" * depth_curly
+
+
+def _load(text: str) -> dict | None:
+    for candidate in (text, _balance(text)):
+        try:
+            value = json.loads(candidate)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(value, dict):
+            return value
+    return None
+
+
+def _unwrap(payload: dict, fallback_name: str | None) -> tuple[str | None, dict]:
+    """Pull a (name, args) pair out of assorted self-describing shapes."""
+    name = fallback_name
+    for key in _NAME_KEYS:
+        value = payload.get(key)
+        if isinstance(value, str) and value:
+            name = value
+            break
+        if isinstance(value, dict) and isinstance(value.get("name"), str):
+            name = value["name"]
+            payload = {**payload, **value}
+            break
+    args: dict = {}
+    for key in _ARG_CONTAINERS:
+        inner = payload.get(key)
+        if isinstance(inner, dict):
+            args = dict(inner)
+            break
+        if isinstance(inner, str):
+            parsed = _load(inner)
+            if parsed is not None:
+                args = parsed
+                break
+    if not args:
+        args = {k: v for k, v in payload.items()
+                if k not in _NAME_KEYS + _ARG_CONTAINERS}
+    return name, args
+
+
+def _first_json_object(text: str) -> str | None:
+    start = text.find("{")
+    return text[start:] if start >= 0 else None
+
+
+def extract_tool_call(message: dict) -> ParsedCall:
+    """The one call to act on, from whichever shape the model used."""
+    calls = message.get("tool_calls") or []
+    if calls:
+        fn = calls[0].get("function", {}) or {}
+        raw = fn.get("arguments")
+        if isinstance(raw, dict):
+            args = raw
+        else:
+            args = _load(raw or "{}") or {}
+        if set(args) & set(_ARG_CONTAINERS):
+            _, args = _unwrap(args, fn.get("name"))
+        return ParsedCall(fn.get("name"), args, "tool_calls")
+
+    content = message.get("content") or ""
+    reasoning = message.get("reasoning_content") or ""
+    for text in (content, reasoning):
+        if not text.strip():
+            continue
+
+        if m := _GLM_CALL.search(text):
+            args = {k.strip(): _coerce(v) for k, v in _GLM_ARG.findall(m.group(2))}
+            return ParsedCall(m.group(1), args, "glm_block")
+
+        bracket = _BRACKET.search(text)
+        blob = _first_json_object(text)
+        if blob is not None:
+            payload = _load(blob)
+            if payload is not None:
+                name, args = _unwrap(payload, bracket.group(1) if bracket else None)
+                if name:
+                    return ParsedCall(name, args, "bracketed" if bracket else "bare_json")
+
+        if bracket and not blob:
+            # A naked "[press_key]" with no arguments is still an instruction.
+            return ParsedCall(bracket.group(1), {}, "bracketed")
+
+    return ParsedCall(None, {}, "none")

benchmarks/edge_harness/android_agent/ui_tree.py

@@ -0,0 +1,146 @@
+"""Turn a uiautomator dump into what a text-only agent can act on.
+
+Spark-X2.5 never sees a screenshot, so the whole agent hinges on this file:
+a screen has to arrive as a short, stable, indexed list of elements, and an
+index the model returns has to map back to a point on the glass.
+
+Two decisions shape the format, both forced by the model rather than by taste:
+
+* **Filter hard.** A real settings screen dumps 150+ nodes, most of them
+  layout containers. At 56 KiB/token of KV cache and a 1.7B model that both
+  costs context and buries the answer, so only nodes that are interactive or
+  carry text survive.
+* **Index, do not coordinate.** The model is asked for `index: 3`, never for
+  pixels -- a text model has no way to know where 3 is on the glass, and
+  coordinates in the transcript would be a source of silent nonsense. The
+  mapping back to a centre point happens here, where the bounds are known.
+
+The rendered form is the one the scenario fixtures were written in, so the
+decision benchmark and the live agent observe identically.
+"""
+
+from __future__ import annotations
+
+import re
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+
+_BOUNDS = re.compile(r"\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]")
+
+# Attributes worth showing when they are true, in the order a reader wants
+# them. `checked` earns its place: the one clearly wrong action in the earlier
+# measurements was tapping a Wi-Fi switch that was already on, which is only
+# avoidable if the state is on the screen the model sees.
+_FLAGS = ("clickable", "checked", "selected", "scrollable", "focused")
+
+
+@dataclass
+class Element:
+    index: int
+    cls: str
+    text: str
+    resource_id: str = ""
+    package: str = ""
+    bounds: tuple[int, int, int, int] = (0, 0, 0, 0)
+    flags: dict[str, bool] = field(default_factory=dict)
+    editable: bool = False
+
+    @property
+    def center(self) -> tuple[int, int]:
+        x1, y1, x2, y2 = self.bounds
+        return (x1 + x2) // 2, (y1 + y2) // 2
+
+    @property
+    def short_class(self) -> str:
+        return self.cls.rsplit(".", 1)[-1] if self.cls else "View"
+
+    def render(self) -> str:
+        parts = [f"[{self.index}]", self.short_class, f'"{self.text}"']
+        for flag in _FLAGS:
+            if flag in self.flags:
+                parts.append(f"{flag}={str(self.flags[flag]).lower()}")
+        return " ".join(parts)
+
+
+def _truthy(node: ET.Element, name: str) -> bool:
+    return node.get(name, "false") == "true"
+
+
+def _label(node: ET.Element) -> str:
+    """What to call this element, preferring what a person would read."""
+    for attr in ("text", "content-desc"):
+        value = (node.get(attr) or "").strip()
+        if value:
+            return value.replace("\n", " ")
+    # A bare resource id is a poor label but far better than an empty string
+    # when the element is the only way forward (unlabelled icon buttons).
+    rid = node.get("resource-id") or ""
+    return rid.rsplit("/", 1)[-1] if rid else ""
+
+
+def _interesting(node: ET.Element) -> bool:
+    if any(_truthy(node, a) for a in ("clickable", "checkable", "scrollable",
+                                      "long-clickable", "focusable")):
+        return True
+    return bool(_label(node)) and not _truthy(node, "password")
+
+
+def parse_bounds(raw: str | None) -> tuple[int, int, int, int]:
+    m = _BOUNDS.search(raw or "")
+    if not m:
+        return (0, 0, 0, 0)
+    return tuple(int(g) for g in m.groups())  # type: ignore[return-value]
+
+
+def parse_ui_dump(xml_text: str, *, max_elements: int = 60) -> list[Element]:
+    """uiautomator XML -> the elements worth showing, in reading order."""
+    root = ET.fromstring(xml_text)
+    elements: list[Element] = []
+    for node in root.iter("node"):
+        if not _interesting(node):
+            continue
+        label = _label(node)
+        clickable = _truthy(node, "clickable")
+        # An unlabelled, non-interactive node tells the model nothing it can
+        # use and costs it context.
+        if not label and not clickable and not _truthy(node, "checkable"):
+            continue
+        flags = {"clickable": clickable}
+        if _truthy(node, "checkable"):
+            flags["checked"] = _truthy(node, "checked")
+        if _truthy(node, "selected"):
+            flags["selected"] = True
+        if _truthy(node, "scrollable"):
+            flags["scrollable"] = True
+        if _truthy(node, "focused"):
+            flags["focused"] = True
+        elements.append(
+            Element(
+                index=len(elements),
+                cls=node.get("class", ""),
+                text=label,
+                resource_id=node.get("resource-id", ""),
+                package=node.get("package", ""),
+                bounds=parse_bounds(node.get("bounds")),
+                flags=flags,
+                editable="EditText" in (node.get("class") or ""),
+            )
+        )
+        if len(elements) >= max_elements:
+            break
+    return elements
+
+
+def render_tree(elements: list[Element]) -> str:
+    """The exact text the model is shown."""
+    return "\n".join(e.render() for e in elements)
+
+
+def screen_signature(elements: list[Element]) -> str:
+    """A cheap identity for a screen, to notice when an action changed nothing.
+
+    Labels and flags only: coordinates shift with animations and scroll offsets,
+    and treating those as a new screen would hide a stuck agent.
+    """
+    return "|".join(f"{e.short_class}:{e.text}:{e.flags.get('checked', '')}"
+                    for e in elements)

benchmarks/edge_harness/archive_profiles.py

@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Archive profiling evidence with byte hashes and reversible large-file compression.
+
+Run only after all writers exit. No benchmark result is changed by this utility.
+"""
+
+import argparse
+import gzip
+import hashlib
+import json
+import shutil
+from pathlib import Path
+
+
+def digest(path, compressed=False):
+    hasher = hashlib.sha256()
+    size = 0
+    opener = gzip.open if compressed else open
+    with opener(path, "rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            hasher.update(chunk)
+            size += len(chunk)
+    return hasher.hexdigest(), size
+
+
+def verify(root):
+    manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8"))
+    for record in manifest["files"]:
+        path = root / record["stored_path"]
+        assert path.resolve().is_relative_to(root.resolve()), "Archive path escapes root"
+        assert digest(path) == (record["stored_sha256"], record["stored_bytes"]), path
+        assert digest(path, record["encoding"] == "gzip") == (record["sha256"], record["bytes"]), path
+    return len(manifest["files"])
+
+
+def archive(source, destination):
+    assert source.resolve() != destination.resolve()
+    assert not destination.resolve().is_relative_to(source.resolve())
+    destination.mkdir(parents=True, exist_ok=False)
+    records = []
+    for path in sorted(source.rglob("*")):
+        relative = path.relative_to(source)
+        if not path.is_file() or "__pycache__" in relative.parts or path.suffix in (".pyc", ".tmp"):
+            continue
+        assert not path.is_symlink(), f"Unexpected evidence symlink: {relative}"
+        original_hash, original_size = digest(path)
+        compress = original_size > 50 * 1024 * 1024 and path.suffix != ".gz"
+        stored = Path("evidence") / relative
+        if compress:
+            stored = stored.with_name(stored.name + ".archive.gz")
+        target = destination / stored
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if compress:
+            with path.open("rb") as reader, target.open("wb") as writer:
+                with gzip.GzipFile(filename="", mode="wb", fileobj=writer, mtime=0) as packed:
+                    shutil.copyfileobj(reader, packed, length=1024 * 1024)
+        else:
+            shutil.copyfile(path, target)
+        stored_hash, stored_size = digest(target)
+        assert digest(path) == (original_hash, original_size), f"Source changed during archive: {relative}"
+        records.append(
+            {
+                "source_path": relative.as_posix(),
+                "stored_path": stored.as_posix(),
+                "sha256": original_hash,
+                "bytes": original_size,
+                "stored_sha256": stored_hash,
+                "stored_bytes": stored_size,
+                "encoding": "gzip" if compress else "identity",
+            }
+        )
+    (destination / "manifest.json").write_text(
+        json.dumps(
+            {
+                "schema": 1,
+                "scope": "Raw evidence bytes; hashes establish integrity, not model correctness or qualification.",
+                "files": records,
+            },
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    (destination / ".gitattributes").write_text("evidence/** -text -whitespace\n", encoding="utf-8")
+    return verify(destination)
+
+
+def restore(root, destination):
+    verify(root)
+    destination.mkdir(parents=True, exist_ok=False)
+    manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8"))
+    for record in manifest["files"]:
+        target = destination / record["source_path"]
+        assert target.resolve().is_relative_to(destination.resolve()), "Restore path escapes root"
+        target.parent.mkdir(parents=True, exist_ok=True)
+        opener = gzip.open if record["encoding"] == "gzip" else open
+        with opener(root / record["stored_path"], "rb") as reader, target.open("wb") as writer:
+            shutil.copyfileobj(reader, writer, length=1024 * 1024)
+        assert digest(target) == (record["sha256"], record["bytes"])
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    sub = parser.add_subparsers(dest="operation", required=True)
+    create = sub.add_parser("create")
+    create.add_argument("source", type=Path)
+    create.add_argument("destination", type=Path)
+    check = sub.add_parser("verify")
+    check.add_argument("archive", type=Path)
+    unpack = sub.add_parser("restore")
+    unpack.add_argument("archive", type=Path)
+    unpack.add_argument("destination", type=Path)
+    args = parser.parse_args()
+    if args.operation == "create":
+        print(f"Archived and verified {archive(args.source, args.destination)} files")
+    elif args.operation == "verify":
+        print(f"Verified {verify(args.archive)} files")
+    else:
+        restore(args.archive, args.destination)
+        print(f"Restored evidence to {args.destination}")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/bench_cpu_vllm.py

@@ -0,0 +1,106 @@
+"""pp512 / tg128 for vLLM on CPU, mirroring `llama-bench` semantics.
+
+pp512: time to process a 512-token prompt (prefill throughput).
+tg128: time to generate 128 tokens (decode throughput), prefill subtracted.
+"""
+import argparse, json, os, sys, time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from bench_harness import provenance  # noqa: E402
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", default="XHToken/Spark-X2.5-1.7B")
+    ap.add_argument("--dtype", default="bfloat16")
+    ap.add_argument("--pp", type=int, default=512)
+    ap.add_argument("--tg", type=int, default=128)
+    ap.add_argument("--reps", type=int, default=3)
+    ap.add_argument("--max-model-len", type=int, default=2048)
+    ap.add_argument("--enforce-eager", action="store_true")
+    ap.add_argument("--quantization", default=None)
+    ap.add_argument("--tag", default="vllm")
+    ap.add_argument("--no-detokenize", action="store_true",
+                    help="skip incremental detokenization, as llama-bench does")
+    ap.add_argument("--compilation-config", default=None,
+                    help="JSON passed straight to LLM(compilation_config=...)")
+    ap.add_argument("--out", default="vllm_cpu_bench.json")
+    a = ap.parse_args()
+
+    import vllm_omni  # noqa: F401
+    from vllm import LLM, SamplingParams
+
+    t0 = time.perf_counter()
+    # Prefix caching must be off: the pp measurement replays one prompt, and a
+    # cache hit would report a prefill that never happened. llama-bench has no
+    # such cache, so leaving it on would not be a comparison.
+    extra = {}
+    if a.compilation_config:
+        extra["compilation_config"] = json.loads(a.compilation_config)
+    llm = LLM(model=a.model, dtype=a.dtype, max_model_len=a.max_model_len,
+              enforce_eager=a.enforce_eager, max_num_seqs=1,
+              quantization=a.quantization, enable_prefix_caching=False,
+              trust_remote_code=False, disable_log_stats=True, **extra)
+    init_s = time.perf_counter() - t0
+
+    def rss_mb(field):
+        with open("/proc/self/status") as f:
+            for line in f:
+                if line.startswith(field):
+                    return round(int(line.split()[1]) / 1024, 1)
+        return None
+
+    prompt_pp = {"prompt_token_ids": list(range(10, 10 + a.pp))}
+    prompt_tg = {"prompt_token_ids": [10]}
+    det = not a.no_detokenize
+    sp_pre = SamplingParams(temperature=0.0, max_tokens=1, ignore_eos=True,
+                            detokenize=det)
+    sp_gen = SamplingParams(temperature=0.0, max_tokens=a.tg, ignore_eos=True,
+                            detokenize=det)
+
+    def timed(prompt, sp, reps):
+        best = []
+        for _ in range(reps):
+            s = time.perf_counter()
+            llm.generate([prompt], sp, use_tqdm=False)
+            best.append(time.perf_counter() - s)
+        return best
+
+    timed(prompt_pp, sp_pre, 1)  # warmup
+
+    pp_t = timed(prompt_pp, sp_pre, a.reps)
+    tg_t = timed(prompt_tg, sp_gen, a.reps)
+    tg1_t = timed(prompt_tg, sp_pre, a.reps)  # 1-token prefill overhead
+
+    pp_best, tg_best, tg1_best = min(pp_t), min(tg_t), min(tg1_t)
+    res = {
+        "tag": a.tag, "model": a.model, "dtype": a.dtype,
+        "compilation_config": a.compilation_config,
+        "detokenize": not a.no_detokenize,
+        "quantization": a.quantization,
+        "enforce_eager": a.enforce_eager, "init_s": round(init_s, 2),
+        # Which build, which threads, which commits -- a throughput number
+        # without these cannot be invalidated later, only doubted.
+        "provenance": provenance({
+            "model_path": os.path.realpath(a.model) if os.path.exists(a.model)
+            else a.model,
+            "pp": a.pp, "tg": a.tg, "reps": a.reps,
+            "max_model_len": a.max_model_len,
+        }),
+        "rss_mb": rss_mb("VmRSS:"), "rss_peak_mb": rss_mb("VmHWM:"),
+        "threads_bind": os.environ.get("VLLM_CPU_OMP_THREADS_BIND", ""),
+        "omp_num_threads": os.environ.get("OMP_NUM_THREADS", ""),
+        "pp_tokens": a.pp, "pp_s": round(pp_best, 4),
+        "pp_tps": round(a.pp / pp_best, 2),
+        "tg_tokens": a.tg,
+        "tg_s": round(tg_best - tg1_best, 4),
+        "tg_tps": round((a.tg - 1) / max(tg_best - tg1_best, 1e-9), 2),
+        "raw": {"pp": pp_t, "tg": tg_t, "tg1": tg1_t},
+    }
+    print(json.dumps({k: v for k, v in res.items() if k != "raw"}, indent=2))
+    with open(a.out, "a") as f:
+        f.write(json.dumps(res) + "\n")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/bench_harness.py

@@ -0,0 +1,634 @@
+"""Run benchmark arms so the numbers survive a shared machine.
+
+Every wrong conclusion this study produced came from measurement, not from
+code: a sequential grid that swung the llama.cpp control 86.7 -> 60.4 tok/s
+mid-run, a 3.1 GB "saving" that was really `VmRSS` counting the mmap'd
+checkpoint's pages, and a W4A8/W4A16 ordering that flipped between grids. The
+discipline that fixed each of them is the same, so it lives here instead of
+being re-typed into the next shell script:
+
+* **Rotate the arm order every pass.** Drift hits arm 1 and arm 5 differently,
+  and interleaving alone does not fix it: if every pass runs the arms in the
+  same order, arm 1 is always the one measured first and carries whatever the
+  machine was doing at the start of each pass. Pass ``p`` starts at arm
+  ``(p - 1) mod n``.
+* **Repeat, and report the spread.** A single number from a shared host is a
+  claim about the host, not the code. An arm whose passes disagree by more
+  than `--noise-threshold` is flagged, and the summary says so.
+* **Record how busy the machine was, and refuse when it is.** A pass taken
+  under load is kept and labelled rather than silently averaged in -- but load
+  average is a whole-host number, and this host has 224 cores, so it says
+  nothing about the 24 this benchmark is pinned to. The pre-flight reads
+  per-core utilisation for the *target* cores, and a host lock keeps two
+  benchmarks off the same cores in the first place. Two sessions benchmarking
+  at once wrecked several measurements here, and one grid's control drifted
+  30% mid-run.
+* **Prefer allocation over memory.** `RssAnon` repeats to within 0.5 MB where
+  `VmRSS` moved 1-2 GB between identical runs.
+* **Know which direction is better.** ``best`` is the maximum for a rate and
+  the minimum for memory, init time or latency. Reporting the maximum for every
+  metric published the *worst* pass as the headline memory figure.
+* **Stamp provenance.** A number whose model, build and thread set were not
+  recorded cannot be retracted later, only doubted -- and this study has
+  retracted four.
+* **Give an arm its own compile cache when it changes the graph.** vLLM keys
+  its AOT compile cache on the *config*, so two arms that differ only by an
+  environment variable share a cached artifact -- and if that variable changes
+  the traced graph, later passes load the wrong one. Three passes of a prefill
+  grid died with ``KeyError: 'weight_halves'`` inside an AOT-loaded forward
+  after the first pass saved an artifact for the other path. Set
+  ``VLLM_CACHE_ROOT`` per arm, and note that the first pass of each arm then
+  pays a cold compile, which is why ``init_s`` is not comparable across passes.
+* **Gate on what the host can actually resolve.** ``--check`` compares against
+  a checked-in expectations file, with a band per metric taken from that
+  metric's *observed* spread: 0.2% for ``RssAnon``, ~5% for per-op self time,
+  and nothing for end-to-end throughput, which swings ~30% here and cannot
+  carry a gate at all. A band is an argument about what is measurable, not a
+  tolerance someone picked.
+
+The arms are subprocess invocations of the existing single-run benchmarks, so
+the measurement code stays the one that was verified; this only decides what
+to run, when, and how to read it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+import os
+import platform
+import shlex
+import statistics
+import subprocess
+import time
+from dataclasses import dataclass, field
+
+
+def load_average() -> float:
+    return os.getloadavg()[0]
+
+
+def parse_cores(spec: str) -> list[int]:
+    """'0-23,48-71' -> [0..23, 48..71]"""
+    cores: list[int] = []
+    for part in spec.split(","):
+        part = part.strip()
+        if not part:
+            continue
+        if "-" in part:
+            lo, hi = part.split("-")
+            cores.extend(range(int(lo), int(hi) + 1))
+        else:
+            cores.append(int(part))
+    return sorted(set(cores))
+
+
+@dataclass
+class Arm:
+    tag: str
+    command: list[str]
+    env: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class Observation:
+    tag: str
+    pass_no: int
+    metrics: dict[str, float]
+    load_before: float
+    load_after: float
+    wall_s: float
+    ok: bool
+    error: str = ""
+
+    @property
+    def busy(self) -> bool:
+        return max(self.load_before, self.load_after) > BUSY_LOAD
+
+
+BUSY_LOAD = 8.0
+"""Above this 1-minute load average a pass is labelled contended.
+
+A poor default on its own: load average is whole-host, and on a 224-core
+machine a 24-thread benchmark contributes ~24 to it by itself, so a fixed 8
+labels every pass contended and the label stops meaning anything. ``main()``
+derives it from the target core count instead -- our own threads plus slack --
+and the per-core pre-flight is the guard that actually decides whether to run.
+"""
+
+LOWER_IS_BETTER = (
+    "rss", "mem", "mb", "init", "load_s", "latency", "ms", "_s", "time", "peak",
+)
+"""Substrings that mark a metric where smaller wins. Rates (`tg_tps`, `pp_tps`)
+take the maximum; memory, init time and latency take the minimum."""
+
+
+HOST_LOCK_PATH = os.environ.get("SPARK_EDGE_LOCK", "/tmp/spark_edge_bench.lock")
+"""Host-level lock. It has to live somewhere every session can see, which a
+session-private scratchpad by definition is not -- that is the whole point of
+a host lock. Override with ``SPARK_EDGE_LOCK``."""
+
+CORE_BUSY_PCT = 50.0
+"""A target core sustained above this is a straggler. OpenMP barriers mean the
+slowest thread gates the whole region, so one heavily-loaded core costs far
+more than its share -- which is why this is a separate test from the aggregate
+below rather than a single averaged number."""
+
+STOLEN_PCT = 5.0
+"""Refuse when this much of the target cores' aggregate capacity is going to
+someone else, which bounds the bias on the measurement.
+
+A flat per-core threshold does not work on this host. With 224 cores and
+several users there is always a transient blip somewhere: a 20% rule over a
+single 0.5 s window refused three consecutive launches on cores that were
+idle seconds earlier, which makes the guard useless rather than strict. So
+utilisation is sampled twice and the per-core minimum is taken -- a spike in
+one window does not count as load -- and the decision is made on aggregate
+stolen capacity plus the straggler rule."""
+
+
+def _cpu_times() -> dict[int, tuple[int, int]]:
+    """(busy, total) jiffies per core, from /proc/stat."""
+    out: dict[int, tuple[int, int]] = {}
+    with open("/proc/stat") as f:
+        for line in f:
+            if not line.startswith("cpu") or line.startswith("cpu "):
+                continue
+            parts = line.split()
+            try:
+                cpu = int(parts[0][3:])
+            except ValueError:
+                continue
+            values = [int(v) for v in parts[1:11]]
+            idle = values[3] + values[4]           # idle + iowait
+            total = sum(values)
+            out[cpu] = (total - idle, total)
+    return out
+
+
+def core_utilization(cores: list[int], window_s: float = 0.5) -> dict[int, float]:
+    """Percentage busy per core over ``window_s``."""
+    before = _cpu_times()
+    time.sleep(window_s)
+    after = _cpu_times()
+    usage: dict[int, float] = {}
+    for cpu in cores:
+        if cpu not in before or cpu not in after:
+            continue
+        busy = after[cpu][0] - before[cpu][0]
+        total = after[cpu][1] - before[cpu][1]
+        usage[cpu] = 0.0 if total <= 0 else busy / total * 100.0
+    return usage
+
+
+def preflight(cores: list[int], *, max_busy_pct: float = CORE_BUSY_PCT,
+              max_stolen_pct: float = STOLEN_PCT,
+              window_s: float = 0.75, samples: int = 2) -> dict:
+    """Refuse to start when contention would bias the measurement.
+
+    Two independent windows; a core's load is the *minimum* of them, so a
+    transient spike is not mistaken for a busy core. Then two tests: aggregate
+    stolen capacity (bounds the average bias) and a per-core straggler rule
+    (one slow core gates every OpenMP barrier).
+    """
+    windows = [core_utilization(cores, window_s) for _ in range(max(samples, 1))]
+    usage = {c: min(w.get(c, 0.0) for w in windows) for c in cores
+             if any(c in w for w in windows)}
+    stolen = (sum(usage.values()) / len(usage)) if usage else 0.0
+    stragglers = {c: round(u, 1) for c, u in usage.items() if u > max_busy_pct}
+    return {"cores": cores,
+            "max_busy_pct": max_busy_pct, "max_stolen_pct": max_stolen_pct,
+            "utilization": {c: round(u, 1) for c, u in usage.items()},
+            "stolen_pct": round(stolen, 2),
+            "busy_cores": stragglers,
+            "clear": stolen <= max_stolen_pct and not stragglers}
+
+
+class HostLock:
+    """An advisory lock so two sessions do not benchmark the same cores.
+
+    Cooperative, and says so: it excludes other runs that take it, not the
+    unrelated job someone starts by hand. That is what the pre-flight is for.
+    """
+
+    def __init__(self, path: str = HOST_LOCK_PATH, note: str = ""):
+        self.path, self.note, self._fd = path, note, None
+
+    def __enter__(self) -> "HostLock":
+        import fcntl
+
+        self._fd = open(self.path, "a+")
+        try:
+            fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            self._fd.seek(0)
+            holder = self._fd.read().strip()[-200:]
+            self._fd.close()
+            self._fd = None
+            raise SystemExit(
+                f"another benchmark holds {self.path}: {holder or 'unknown'}. "
+                f"Wait for it, or set SPARK_EDGE_LOCK to a different path if "
+                f"you really are on disjoint cores."
+            )
+        self._fd.seek(0)
+        self._fd.truncate()
+        self._fd.write(f"pid={os.getpid()} {time.strftime('%H:%M:%S')} {self.note}\n")
+        self._fd.flush()
+        return self
+
+    def __exit__(self, *exc) -> None:
+        if self._fd is not None:
+            import fcntl
+
+            fcntl.flock(self._fd, fcntl.LOCK_UN)
+            self._fd.close()
+            self._fd = None
+
+
+DEFAULT_BANDS_PCT = {
+    "rss_anon_mb": 1.5,
+    "peak_rss_anon_mb": 2.0,
+    "rss_anon_at_context_mb": 2.0,
+    "init_s": 12.0,
+    "ready_s": 12.0,
+    "import_s": 12.0,
+    "step_ms": 6.0,
+    "gemm_ms": 6.0,
+}
+"""Regression bands, per metric, from the spread each one actually shows on
+this host. ``RssAnon`` repeats to 0.2% and is given 1.5% for safety; init time
+is dominated by page cache and repeats far less tightly; per-op self time
+reproduces to ~5%.
+
+Throughput is deliberately absent. ``tg_tps`` swings ~30% run to run on this
+shared host, so any band wide enough not to fire constantly is too wide to
+catch a real regression. Gate on memory and per-op time instead; that is the
+whole reason the per-op profiler exists."""
+
+UNGATEABLE = {"tg_tps", "pp_tps", "tps"}
+"""Metrics this host cannot resolve well enough to gate on."""
+
+
+def check_expectations(summaries: list["ArmSummary"], expected: dict) -> list[dict]:
+    """Compare measured medians against a checked-in expectations file.
+
+    The median, not the best: `best` is one pass and a gate that fires on the
+    luckiest run of a noisy metric is a coin toss.
+    """
+    bands = {**DEFAULT_BANDS_PCT, **(expected.get("bands_pct") or {})}
+    arms = expected.get("arms") or {}
+    results = []
+    for s in summaries:
+        want = (arms.get(s.tag) or {}).get(s.metric)
+        if want is None:
+            results.append({"arm": s.tag, "metric": s.metric, "status": "unlisted",
+                            "measured": s.median, "expected": None})
+            continue
+        if s.metric in UNGATEABLE:
+            results.append({"arm": s.tag, "metric": s.metric, "status": "ungateable",
+                            "measured": s.median, "expected": want,
+                            "why": "this host cannot resolve it"})
+            continue
+        band = bands.get(s.metric, 5.0)
+        drift = abs(s.median - want) / want * 100.0 if want else 0.0
+        worse = (s.median > want) if s.lower_is_better else (s.median < want)
+        results.append({
+            "arm": s.tag, "metric": s.metric,
+            "measured": s.median, "expected": want,
+            "drift_pct": round(drift, 2), "band_pct": band,
+            "status": ("ok" if drift <= band else
+                       "REGRESSION" if worse else "improved"),
+        })
+    return results
+
+
+def format_check(results: list[dict]) -> str:
+    if not results:
+        return "nothing to check"
+    width = max(len(r["arm"]) for r in results)
+    lines = []
+    for r in results:
+        if r["status"] in ("unlisted", "ungateable"):
+            lines.append(f"  {r['arm'].ljust(width)}  {r['metric']:22s} "
+                         f"{r['measured']:9.2f}  {r['status']}"
+                         + (f" ({r['why']})" if r.get("why") else ""))
+        else:
+            lines.append(f"  {r['arm'].ljust(width)}  {r['metric']:22s} "
+                         f"{r['measured']:9.2f}  vs {r['expected']:9.2f}  "
+                         f"{r['drift_pct']:+5.2f}% (band {r['band_pct']:.1f}%)  "
+                         f"{r['status']}")
+    return "\n".join(lines)
+
+
+def metric_is_lower_better(metric: str) -> bool:
+    name = metric.lower()
+    if name.endswith(("_tps", "_tok_s", "tps")):
+        return False
+    return any(tok in name for tok in LOWER_IS_BETTER)
+
+
+def provenance(extra: dict | None = None) -> dict:
+    """Everything needed to decide later whether a number is still valid.
+
+    Saved profiles in this tree carry no model path, context length or build,
+    so two of them are indistinguishable from each other and one set silently
+    describes a checkpoint that was later found wrong. This is the fix.
+    """
+    root = os.environ.get("EMBEDDING_INFER_ROOT", "/data/zhoutaichang/embedding_infer")
+
+    def _sha(rel: str) -> str | None:
+        try:
+            out = subprocess.run(["git", "-C", os.path.join(root, rel),
+                                  "rev-parse", "HEAD"],
+                                 capture_output=True, text=True, timeout=10)
+            return out.stdout.strip() or None
+        except Exception:
+            return None
+
+    try:
+        affinity = sorted(os.sched_getaffinity(0))
+    except AttributeError:
+        affinity = []
+    return {
+        "recorded_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+        "host": platform.node(),
+        "python": platform.python_version(),
+        "cpu_count": os.cpu_count(),
+        "n_affinity": len(affinity),
+        "affinity": affinity,
+        "commits": {
+            "embedding_infer": _sha("."),
+            "vllm_omni": _sha("vllm-omni"),
+            "vllm": _sha("vllm"),
+        },
+        "env": {k: v for k, v in sorted(os.environ.items())
+                if k.startswith(("OMP_", "KMP_", "MKL_", "VLLM_", "LD_PRELOAD",
+                                 "CUDA_VISIBLE_DEVICES"))},
+        "loadavg": os.getloadavg(),
+        **(extra or {}),
+    }
+
+
+def run_once(arm: Arm, pass_no: int, timeout: float, keys: list[str]) -> Observation:
+    env = {**os.environ, **arm.env}
+    load_before = load_average()
+    t0 = time.perf_counter()
+    proc = subprocess.run(arm.command, env=env, capture_output=True,
+                          text=True, timeout=timeout)
+    wall = time.perf_counter() - t0
+    load_after = load_average()
+    if proc.returncode != 0:
+        return Observation(arm.tag, pass_no, {}, load_before, load_after, wall,
+                           ok=False, error=proc.stderr.strip()[-400:])
+    metrics = _scrape(proc.stdout, keys)
+    return Observation(arm.tag, pass_no, metrics, load_before, load_after, wall,
+                       ok=bool(metrics))
+
+
+def _scrape(stdout: str, keys: list[str]) -> dict[str, float]:
+    """Pull metrics out of the last JSON object the run printed."""
+    start = stdout.rfind("{")
+    while start >= 0:
+        try:
+            payload = json.loads(stdout[start:stdout.rfind("}") + 1])
+        except json.JSONDecodeError:
+            start = stdout.rfind("{", 0, start)
+            continue
+        return {k: float(payload[k]) for k in keys
+                if isinstance(payload.get(k), (int, float))}
+    return {}
+
+
+@dataclass
+class ArmSummary:
+    tag: str
+    metric: str
+    best: float
+    median: float
+    spread_pct: float
+    n: int
+    n_busy: int
+    lower_is_better: bool = False
+    noise_threshold: float = 8.0
+
+    @property
+    def noisy(self) -> bool:
+        return self.spread_pct > self.noise_threshold
+
+
+NOISE_THRESHOLD = 8.0
+
+
+def summarize(observations: list[Observation], metric: str,
+              noise_threshold: float = NOISE_THRESHOLD,
+              lower_is_better: bool | None = None) -> list[ArmSummary]:
+    """Collapse passes into one row per arm.
+
+    ``lower_is_better`` defaults to reading the metric name, so `rss_anon_mb`
+    and `init_s` take their minimum while `tg_tps` takes its maximum. Taking
+    the maximum for every metric is how the *worst* memory pass came to be
+    published as the best one.
+    """
+    if lower_is_better is None:
+        lower_is_better = metric_is_lower_better(metric)
+    by_tag: dict[str, list[Observation]] = {}
+    for obs in observations:
+        if obs.ok and metric in obs.metrics:
+            by_tag.setdefault(obs.tag, []).append(obs)
+    summaries = []
+    for tag, rows in by_tag.items():
+        values = [r.metrics[metric] for r in rows]
+        lo, hi = min(values), max(values)
+        spread = 0.0 if hi == 0 else (hi - lo) / hi * 100.0
+        summaries.append(ArmSummary(
+            tag=tag, metric=metric, best=lo if lower_is_better else hi,
+            median=statistics.median(values), spread_pct=spread,
+            n=len(values), n_busy=sum(1 for r in rows if r.busy),
+            lower_is_better=lower_is_better, noise_threshold=noise_threshold,
+        ))
+    return sorted(summaries, key=lambda s: s.best if lower_is_better else -s.best)
+
+
+def indistinguishable(summaries: list[ArmSummary]) -> str | None:
+    """Say so when the gap between the top two arms is inside the noise.
+
+    Per-op self time resolves ~5% on this host; end-to-end throughput resolves
+    ~30%. An 8% difference in tg_tps is not a result, and publishing it as one
+    is how the W4A8/W4A16 ordering came to flip between grids.
+    """
+    if len(summaries) < 2:
+        return None
+    first, second = summaries[0], summaries[1]
+    if not first.best:
+        return None
+    gap = abs(first.best - second.best) / abs(first.best) * 100.0
+    band = max(first.spread_pct, second.spread_pct,
+               DEFAULT_BANDS_PCT.get(first.metric, first.noise_threshold))
+    if gap > band:
+        return None
+    return (f"{first.tag} and {second.tag} differ by {gap:.1f}% on "
+            f"{first.metric}, inside this measurement's own spread "
+            f"({band:.1f}%). That is not a ranking. For a difference this "
+            f"size use per-op self time, which reproduces to ~5%: "
+            f"profile_cpu_step.py --model <build> --prompt-len <ctx>")
+
+
+def format_summary(summaries: list[ArmSummary]) -> str:
+    if not summaries:
+        return "no successful observations"
+    width = max(len(s.tag) for s in summaries)
+    direction = "min" if summaries[0].lower_is_better else "max"
+    lines = [f"{'arm'.ljust(width)}  {f'best({direction})':>9s} {'median':>9s} "
+             f"{'spread':>7s}  n  notes"]
+    for s in summaries:
+        notes = []
+        if s.noisy:
+            notes.append(f"NOISY (>{s.noise_threshold:.0f}%)")
+        if s.n_busy:
+            notes.append(f"{s.n_busy} pass(es) under load")
+        if s.n < 2:
+            notes.append("single pass — not a result")
+        lines.append(f"{s.tag.ljust(width)}  {s.best:9.2f} {s.median:9.2f} "
+                     f"{s.spread_pct:6.1f}%  {s.n}  {'; '.join(notes)}")
+    warning = indistinguishable(summaries)
+    if warning:
+        lines.append("")
+        lines.append(f"  NOT A RANKING: {warning}")
+    return "\n".join(lines)
+
+
+def run_grid(arms: list[Arm], *, passes: int, metric: str, timeout: float,
+             keys: list[str], settle_s: float = 0.0,
+             busy_load: float = BUSY_LOAD) -> list[Observation]:
+    """Run every arm each pass, rotating the starting arm between passes.
+
+    Interleaving alone leaves arm 0 permanently first, so whatever the machine
+    does at the start of a pass always lands on the same arm. Pass ``p`` starts
+    at arm ``(p - 1) mod n``.
+    """
+    globals()["BUSY_LOAD"] = busy_load
+    observations: list[Observation] = []
+    for p in range(1, passes + 1):
+        shift = (p - 1) % len(arms) if arms else 0
+        for arm in arms[shift:] + arms[:shift]:
+            if settle_s:
+                time.sleep(settle_s)
+            obs = run_once(arm, p, timeout, keys)
+            observations.append(obs)
+            status = "ok" if obs.ok else f"FAILED: {obs.error[:80]}"
+            value = obs.metrics.get(metric)
+            shown = f"{value:.2f}" if value is not None else "-"
+            flag = "  [busy]" if obs.busy else ""
+            print(f"  pass {p} {arm.tag:24s} {metric}={shown:>9s}  "
+                  f"load {obs.load_before:.1f}->{obs.load_after:.1f}{flag}  {status}",
+                  flush=True)
+    return observations
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--spec", required=True,
+                    help="JSON file: {\"arms\": [{\"tag\":..., \"command\":[...], \"env\":{...}}]}")
+    ap.add_argument("--passes", type=int, default=3)
+    ap.add_argument("--metric", default="tg_tps")
+    ap.add_argument("--keys", default="tg_tps,pp_tps,init_s,rss_anon_mb,rss_mb")
+    ap.add_argument("--timeout", type=float, default=2400)
+    ap.add_argument("--settle-s", type=float, default=2.0)
+    ap.add_argument("--busy-load", type=float, default=None,
+                    help="load average above which a pass is labelled "
+                         "contended; defaults to (target cores + 8), since "
+                         "the run itself contributes about one per core")
+    ap.add_argument("--noise-threshold", type=float, default=NOISE_THRESHOLD)
+    ap.add_argument("--out", default=None)
+    ap.add_argument("--cores", default="",
+                    help="target cores, e.g. '0-23'. Checked for other "
+                         "people's work before the run starts.")
+    ap.add_argument("--allow-busy", action="store_true",
+                    help="run even if the target cores are already loaded "
+                         "(the result is labelled, not trusted)")
+    ap.add_argument("--check", default=None,
+                    help="expectations JSON; exit non-zero on a regression "
+                         "outside the metric's measured band")
+    ap.add_argument("--no-lock", action="store_true",
+                    help="skip the host lock (only if you are on cores no "
+                         "other benchmark uses)")
+    a = ap.parse_args()
+
+    spec = json.loads(open(a.spec).read())
+    arms = [Arm(tag=x["tag"],
+                command=x["command"] if isinstance(x["command"], list)
+                else shlex.split(x["command"]),
+                env={k: str(v) for k, v in (x.get("env") or {}).items()})
+            for x in spec["arms"]]
+    keys = [k.strip() for k in a.keys.split(",") if k.strip()]
+
+    cores = parse_cores(a.cores) if a.cores else sorted(os.sched_getaffinity(0))
+    # Our own run puts ~len(cores) on the load average, so a threshold below
+    # that labels every pass contended and tells us nothing.
+    busy_load = a.busy_load if a.busy_load is not None else len(cores) + 8.0
+    check = preflight(cores)
+    if not check["clear"]:
+        detail = []
+        if check["stolen_pct"] > check["max_stolen_pct"]:
+            detail.append(f"{check['stolen_pct']}% of the cores' capacity is "
+                          f"someone else's (limit {check['max_stolen_pct']}%)")
+        if check["busy_cores"]:
+            detail.append("sustained stragglers: "
+                          + ", ".join(f"{c}={p}%"
+                                      for c, p in check["busy_cores"].items()))
+        msg = "contended target cores: " + "; ".join(detail)
+        if not a.allow_busy:
+            raise SystemExit(
+                msg + "\n  Another job is on these cores; this run would "
+                      "measure the contention, not the code. Wait, pick other "
+                      "cores with --cores, or pass --allow-busy to label the "
+                      "result instead of trusting it.")
+        print(f"WARNING: {msg} (--allow-busy)")
+
+    print(f"{len(arms)} arms x {a.passes} passes, rotating; metric={a.metric}; "
+          f"{len(cores)} cores, pre-flight "
+          f"{'clear' if check['clear'] else 'BUSY'} "
+          f"({check['stolen_pct']}% stolen), "
+          f"busy-label above load {busy_load:.0f}")
+
+    lock = contextlib.nullcontext() if a.no_lock else HostLock(note=a.metric)
+    with lock:
+        observations = run_grid(arms, passes=a.passes, metric=a.metric,
+                                timeout=a.timeout, keys=keys,
+                                settle_s=a.settle_s, busy_load=busy_load)
+    summaries = summarize(observations, a.metric, a.noise_threshold)
+    print("\n" + format_summary(summaries))
+
+    checks = []
+    if a.check:
+        checks = check_expectations(summaries, json.loads(open(a.check).read()))
+        print("\nregression check\n" + format_check(checks))
+
+    if a.out:
+        with open(a.out, "w") as f:
+            json.dump({
+                "metric": a.metric, "passes": a.passes,
+                "lower_is_better": metric_is_lower_better(a.metric),
+                "provenance": provenance({
+                    "arms": [{"tag": x.tag, "command": x.command, "env": x.env}
+                             for x in arms],
+                    "preflight": check,
+                    "busy_load_threshold": busy_load,
+                }),
+                "observations": [vars(o) for o in observations],
+                "summary": [vars(s) for s in summaries],
+                "checks": checks,
+            }, f, indent=2)
+        print(f"\nwrote {a.out}")
+
+    regressions = [c for c in checks if c["status"] == "REGRESSION"]
+    if regressions:
+        raise SystemExit(
+            f"{len(regressions)} regression(s): "
+            + ", ".join(f"{c['arm']}/{c['metric']} {c['drift_pct']:+.2f}%"
+                        for c in regressions))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/bench_init_memory.py

@@ -0,0 +1,297 @@
+"""Init time and resident memory, which decide whether a model can be served
+from an edge box at all.
+
+Three quantities, because the plan's items are defined in terms of them and
+the earlier version of this file measured none of them:
+
+* **ready_s -- process start to serving.** The old ``init_s`` started its timer
+  *after* ``import vllm_omni``, so the 14.4 s of import cost the plan wants to
+  cut was invisible to the benchmark that was supposed to prove the cut. The
+  clock now starts at process creation, read from ``/proc/self/stat``, so
+  interpreter start-up and imports are inside it. ``init_s`` is still reported,
+  unchanged in meaning, so older rows stay comparable.
+* **peak_rss_anon_mb -- the high-water mark, sampled.** What an edge device has
+  to survive is the *peak*, not the steady state: this model loads at 7338 MB
+  and settles at 3510 MB only after the allocator is asked for its arenas back,
+  and a device that cannot hold the peak never reaches the release. There is no
+  kernel high-water mark for RssAnon (``VmHWM`` and ``ru_maxrss`` both track
+  VmRSS, which counts the mmap'd checkpoint and moved 1-2 GB between identical
+  runs), so a sampler thread takes it.
+* **rss_anon_at_context_mb -- memory once the cache is actually populated.**
+  The old default generated 8 tokens, so the KV cache was allocated and never
+  touched; "KV sizing saves only 188 MB" was measured there. ``--context-len``
+  fills it.
+
+And one correction that changes what the older rows mean. Everything here used
+to read ``/proc/self/status``, i.e. **this process only**. That is the right
+number only when the model is in-process, which happens when
+``VLLM_ENABLE_V1_MULTIPROCESSING=0`` keeps the executor at ``uni``
+(``vllm/platforms/cpu.py:273`` promotes it to ``mp`` otherwise). Every earlier
+run set that variable, so the published figures are in-process figures. Run the
+same benchmark at vLLM's defaults and the weights land in a worker process the
+benchmark never looked at: 796 MB "resident" for a model whose weights alone
+are 1.4 GB. Resident memory is now summed over the **process tree**, which is
+what an edge device actually has to hold, and both numbers are reported so the
+older rows remain comparable.
+
+Every row carries provenance, so it can be invalidated later instead of merely
+doubted.
+"""
+import time
+
+_T_IMPORT = time.time()
+
+import argparse, json, os, resource, sys, threading  # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from bench_harness import provenance  # noqa: E402
+
+
+def _rss(field: str, pid: str | int = "self") -> float | None:
+    try:
+        with open(f"/proc/{pid}/status") as f:
+            for line in f:
+                if line.startswith(field):
+                    return round(int(line.split()[1]) / 1024, 1)
+    except OSError:
+        return None
+    return None
+
+
+def _descendants(root: int) -> list[int]:
+    """Every process under ``root``. vLLM's default CPU path runs the model in
+    a worker process, so a self-only reading misses the weights entirely."""
+    children: dict[int, list[int]] = {}
+    for entry in os.listdir("/proc"):
+        if not entry.isdigit():
+            continue
+        try:
+            with open(f"/proc/{entry}/stat") as f:
+                fields = f.read().rsplit(")", 1)[1].split()
+            ppid = int(fields[1])
+        except (OSError, IndexError, ValueError):
+            continue
+        children.setdefault(ppid, []).append(int(entry))
+    out, stack = [], [root]
+    while stack:
+        pid = stack.pop()
+        for child in children.get(pid, []):
+            out.append(child)
+            stack.append(child)
+    return out
+
+
+def _tree_anon_mb(pids: list[int]) -> float:
+    return round(sum(_rss("RssAnon:", p) or 0.0 for p in pids), 1)
+
+
+def _process_start_epoch() -> float | None:
+    """When this process was created, so imports are inside the measurement."""
+    try:
+        with open("/proc/self/stat") as f:
+            fields = f.read().rsplit(")", 1)[1].split()
+        starttime_ticks = int(fields[19])          # /proc/self/stat field 22
+        hz = os.sysconf("SC_CLK_TCK")
+        with open("/proc/stat") as f:
+            for line in f:
+                if line.startswith("btime"):
+                    return int(line.split()[1]) + starttime_ticks / hz
+    except Exception:
+        return None
+    return None
+
+
+class AnonSampler(threading.Thread):
+    """Peak RssAnon has no kernel counter, so poll for it -- across the whole
+    process tree, because the model may not be in this process."""
+
+    def __init__(self, interval: float = 0.02, rescan_every: float = 0.5):
+        super().__init__(daemon=True)
+        self.interval, self.rescan_every = interval, rescan_every
+        self.peak = self.peak_self = 0.0
+        self.marks: list[dict] = []
+        self.pids = [os.getpid()]
+        self.max_procs = 1
+        # not `_stop`: threading.Thread already defines a private _stop().
+        self._done = threading.Event()
+
+    def _refresh(self) -> None:
+        me = os.getpid()
+        self.pids = [me] + _descendants(me)
+        self.max_procs = max(self.max_procs, len(self.pids))
+
+    def _sample(self) -> tuple[float, float]:
+        mine = _rss("RssAnon:") or 0.0
+        tree = _tree_anon_mb(self.pids)
+        self.peak_self = max(self.peak_self, mine)
+        self.peak = max(self.peak, tree)
+        return mine, tree
+
+    def run(self) -> None:
+        last_scan = 0.0
+        while not self._done.is_set():
+            now = time.monotonic()
+            if now - last_scan >= self.rescan_every:
+                self._refresh()
+                last_scan = now
+            self._sample()
+            self._done.wait(self.interval)
+
+    def mark(self, name: str) -> float | None:
+        """Record a phase boundary with the peak reached so far."""
+        self._refresh()
+        mine, tree = self._sample()
+        self.marks.append({"phase": name, "anon_mb": mine,
+                           "tree_anon_mb": tree, "n_procs": len(self.pids),
+                           "peak_tree_so_far_mb": round(self.peak, 1)})
+        return mine
+
+    def stop(self) -> None:
+        self._done.set()
+        self.join(timeout=2.0)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--tag", required=True)
+    ap.add_argument("--max-model-len", type=int, default=2048)
+    ap.add_argument("--quantization", default=None)
+    ap.add_argument("--kv-cache-memory-bytes", type=int, default=None,
+                    help="exact KV budget; VLLM_CPU_KVCACHE_SPACE is integer GiB "
+                         "so it cannot express an edge-sized cache")
+    ap.add_argument("--max-num-batched-tokens", type=int, default=None,
+                    help="vLLM sizes its warm-up forward from this; an edge "
+                         "deployment serving one short stream does not need "
+                         "the server default")
+    ap.add_argument("--trim", action="store_true",
+                    help="return freed allocator arenas to the OS after init")
+    ap.add_argument("--generate", type=int, default=8,
+                    help="tokens to generate, to prove the engine really works")
+    ap.add_argument("--context-len", type=int, default=0,
+                    help="after init, run a request with this many prompt "
+                         "tokens so the KV cache is populated, and report "
+                         "resident memory there. 0 skips it -- and a memory "
+                         "number taken without it says nothing about cache "
+                         "sizing.")
+    ap.add_argument("--sample-interval", type=float, default=0.02)
+    ap.add_argument("--out", required=True)
+    a = ap.parse_args()
+
+    proc_start = _process_start_epoch()
+    sampler = AnonSampler(a.sample_interval)
+    sampler.start()
+
+    anon_start = sampler.mark("before_import")
+    import vllm_omni  # noqa: F401
+    from vllm import LLM, SamplingParams
+
+    anon_after_import = sampler.mark("after_import")
+    import_s = time.time() - _T_IMPORT
+
+    t0 = time.perf_counter()
+    extra = {}
+    if a.kv_cache_memory_bytes:
+        extra["kv_cache_memory_bytes"] = a.kv_cache_memory_bytes
+    if a.max_num_batched_tokens:
+        extra["max_num_batched_tokens"] = a.max_num_batched_tokens
+    llm = LLM(model=a.model, dtype="bfloat16", max_model_len=a.max_model_len,
+              max_num_seqs=1, quantization=a.quantization,
+              enable_prefix_caching=False, trust_remote_code=False,
+              disable_log_stats=True, **extra)
+    init_s = time.perf_counter() - t0
+    ready_s = (time.time() - proc_start) if proc_start else None
+
+    anon_after_engine = sampler.mark("after_engine")
+    peak_during_load = round(sampler.peak, 1)
+
+    trimmed = None
+    if a.trim:
+        # The GPTQ repack unpacks 4-bit weights to int32 (4 bytes per weight)
+        # and throws the intermediates away, but a caching allocator keeps the
+        # arenas, so they stay resident. Ask for them back.
+        import ctypes
+
+        released = []
+        for lib, sym in ((None, "malloc_trim"),
+                         ("libtcmalloc_minimal.so.4",
+                          "MallocExtension_ReleaseFreeMemory")):
+            try:
+                handle = ctypes.CDLL(lib) if lib else ctypes.CDLL("libc.so.6")
+                fn = getattr(handle, sym)
+                fn(0) if sym == "malloc_trim" else fn()
+                released.append(sym)
+            except Exception:
+                pass
+        trimmed = released
+    anon_after_trim = sampler.mark("after_trim")
+
+    out = llm.generate([{"prompt_token_ids": [10]}],
+                       SamplingParams(temperature=0.0, max_tokens=a.generate,
+                                      ignore_eos=True), use_tqdm=False)
+    n_out = len(out[0].outputs[0].token_ids)
+    sampler.mark("after_short_generate")
+
+    anon_at_context = context_tokens = None
+    if a.context_len:
+        # Fill the cache. Without this the cache is allocated and never
+        # written, so resident memory says nothing about how it was sized.
+        context_tokens = min(a.context_len, a.max_model_len - 16)
+        llm.generate([{"prompt_token_ids": [10] * context_tokens}],
+                     SamplingParams(temperature=0.0, max_tokens=16,
+                                    ignore_eos=True), use_tqdm=False)
+        anon_at_context = sampler.mark("after_context_generate")
+
+    sampler.stop()
+
+    rec = {
+        "tag": a.tag, "model": a.model,
+        "ready_s": round(ready_s, 2) if ready_s else None,
+        "import_s": round(import_s, 2),
+        "init_s": round(init_s, 2),
+        # VmRSS counts the mmap'd checkpoint's file pages, which come and go
+        # with system page-cache pressure: identical configurations measured
+        # 1.2-2.0 GB apart. RssAnon is the allocated memory -- weights copied
+        # into tensors, the KV cache, activations -- and is what a memory
+        # budget is actually about.
+        # self-only, for comparability with rows taken before the tree fix
+        "rss_anon_mb": _rss("RssAnon:"), "rss_file_mb": _rss("RssFile:"),
+        # what the device actually has to hold
+        "rss_anon_tree_mb": _tree_anon_mb([os.getpid()] + _descendants(os.getpid())),
+        "peak_rss_anon_mb": round(sampler.peak, 1),
+        "peak_rss_anon_self_mb": round(sampler.peak_self, 1),
+        "peak_rss_anon_during_load_mb": peak_during_load,
+        "n_procs_max": sampler.max_procs,
+        "anon_before_import_mb": anon_start,
+        "anon_after_import_mb": anon_after_import,
+        "anon_after_engine_mb": anon_after_engine,
+        "anon_after_trim_mb": anon_after_trim,
+        "rss_anon_at_context_mb": anon_at_context,
+        "context_tokens": context_tokens,
+        "phase_marks": sampler.marks,
+        "trim_calls": trimmed,
+        "rss_mb": _rss("VmRSS:"), "rss_peak_mb": _rss("VmHWM:"),
+        "maxrss_mb": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1),
+        "kvcache_space_gib": os.environ.get("VLLM_CPU_KVCACHE_SPACE", ""),
+        "kv_cache_memory_bytes": a.kv_cache_memory_bytes,
+        "max_num_batched_tokens": a.max_num_batched_tokens,
+        "cache_root": os.environ.get("VLLM_CACHE_ROOT", "(default)"),
+        "aot_shim": os.environ.get("VLLM_OMNI_AOT_CACHE_SHIM", "1"),
+        "max_model_len": a.max_model_len,
+        "tokens_generated": n_out,
+        "provenance": provenance({
+            "quantization": a.quantization,
+            "sample_interval_s": a.sample_interval,
+            # uni vs mp decides whether the model is in this process at all
+            "v1_multiprocessing": os.environ.get(
+                "VLLM_ENABLE_V1_MULTIPROCESSING", "1"),
+            "memory_scope": "process tree",
+        }),
+    }
+    print(json.dumps(rec, indent=2))
+    with open(a.out, "a") as f:
+        f.write(json.dumps(rec) + "\n")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/check_text_reliability.py

@@ -0,0 +1,175 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Isolated real-model stream saturation, cancellation and owned-worker failure checks."""
+
+import argparse
+import asyncio
+import copy
+import os
+import sys
+import time
+import traceback
+from pathlib import Path
+
+from profile_local_text import save
+
+
+async def run(args):
+    import psutil
+
+    from vllm_omni.edge.local.engine import LocalTextEngine
+    from vllm_omni.edge.local.plan import plan_text_session
+
+    args.out.mkdir(parents=True, exist_ok=False)
+    report = {
+        "status": "running",
+        "start_unix": time.time(),
+        "argv": sys.argv,
+        "scope": (
+            "LocalTextEngine consumer queue and actual owned engine-process failure; "
+            "not proof of every internal queue being bounded"
+        ),
+    }
+    engine = None
+    owned = []
+    save(args.out / "report.json", report)
+    try:
+        plan = plan_text_session(
+            args.model, max_model_len=4096, max_num_seqs=4, max_num_batched_tokens=512, enforce_eager=True
+        )
+        report["plan"] = plan.to_dict()
+        if not plan.admitted:
+            report["status"] = "rejected"
+            return
+        oversized = plan_text_session(
+            args.model, max_model_len=4096, max_num_seqs=100000, max_num_batched_tokens=512, enforce_eager=True
+        )
+        report["oversized_plan"] = oversized.to_dict()
+        report["oversized_plan_refused"] = not oversized.admitted
+        engine = LocalTextEngine(plan)
+        await engine.start()
+        prompt = "Describe a quiet garden with trees, flowers and a pond."
+
+        async def request(saturate=False):
+            session = engine.open_session()
+            rid, stream = await engine.submit(
+                session, prompt, max_tokens=128, temperature=0, ignore_eos=True, max_chunks=4
+            )
+            if saturate:
+                # Wait for actual capacity, rather than guessing a delay from device speed.
+                deadline = time.monotonic() + 30
+                while stream.stats()["high_water_chunks"] < 4 and time.monotonic() < deadline:
+                    await asyncio.sleep(0.05)
+                report["saturated_queue_before_drain"] = stream.stats()
+            events = [e.to_dict() async for e in stream]
+            record = engine.records[rid].to_dict()
+            record["stream"] = stream.stats()
+            record["events"] = events
+            engine.close_session(session.session_id)
+            assert record["finished"] and not record["error"] and record["output_tokens"] == 128
+            return record
+
+        report["baseline"] = await request()
+        report["saturated"] = await request(saturate=True)
+        report["saturation_reached"] = report["saturated_queue_before_drain"]["high_water_chunks"] == 4
+        report["same_tokens_after_saturation"] = (
+            report["baseline"]["output_token_ids"] == report["saturated"]["output_token_ids"]
+        )
+        session = engine.open_session()
+        stale = copy.deepcopy(session)
+        rid, stream = await engine.submit(session, prompt, max_tokens=512, ignore_eos=True, max_chunks=4)
+        deadline = time.monotonic() + 30
+        while stream.stats()["high_water_chunks"] < 4 and time.monotonic() < deadline:
+            await asyncio.sleep(0.05)
+        report["cancel_queue_before"] = stream.stats()
+        report["cancel"] = await engine.cancel(rid)
+        report["cancel"]["late_events"] = [e.to_dict() async for e in stream]
+        try:
+            await engine.submit(stale, prompt, max_tokens=1)
+        except RuntimeError as error:
+            report["stale_handle_rejected"] = str(error)
+        else:
+            report["stale_handle_rejected"] = False
+        engine.close_session(session.session_id)
+        report["after_cancel"] = await request()
+        save(args.out / "report.json", report)
+
+        # Process handles come from this engine's actual manager, then must also
+        # belong to this probe's process tree. Never select arbitrary system PIDs.
+        client = engine._omni.engine.stage_clients[0]
+        manager = client.resources.engine_manager
+        processes = [p for p in manager.processes if p.is_alive()]
+        descendants = {p.pid: p for p in psutil.Process().children(recursive=True)}
+        assert processes and all(p.pid in descendants for p in processes)
+        manager_tree = {}
+        for handle in processes:
+            core = descendants[handle.pid]
+            manager_tree[core.pid] = core
+            manager_tree.update({child.pid: child for child in core.children(recursive=True)})
+        owned = [(p.pid, p.create_time()) for p in manager_tree.values()]
+        target = processes[0]
+        session = engine.open_session()
+        rid, stream = await engine.submit(session, prompt, max_tokens=512, ignore_eos=True)
+        first = await asyncio.wait_for(stream.get(), 30)
+        assert first is not None and first.kind == "token", "No token before worker-failure injection"
+        report["crash_first"] = first.to_dict()
+        report["crash_injection"] = {
+            "pid": target.pid,
+            "name": target.name,
+            "created": descendants[target.pid].create_time(),
+            "ownership": "engine manager handle and current process descendant",
+        }
+        start = time.perf_counter()
+        target.terminate()
+
+        async def drain():
+            return [e.to_dict() async for e in stream]
+
+        try:
+            events = await asyncio.wait_for(drain(), 30)
+            report["crash_injection"].update(
+                events=events,
+                observed_within_30s=True,
+                explicit_error_reported=bool(engine.records[rid].error) or any(e["kind"] == "error" for e in events),
+                request_record=engine.records[rid].to_dict(),
+            )
+        except asyncio.TimeoutError:
+            report["crash_injection"].update(observed_within_30s=False, error="No terminal stream event within 30s")
+        report["crash_injection"]["observation_s"] = time.perf_counter() - start
+        report["status"] = "completed"
+    except Exception as error:
+        report.update(status="failed", error=repr(error), traceback=traceback.format_exc())
+        traceback.print_exc()
+    finally:
+        if engine is not None:
+            try:
+                await engine.close()
+            except (Exception, asyncio.CancelledError) as error:
+                report.update(status="failed", cleanup_error=repr(error))
+                report.setdefault("error", f"Engine shutdown failed after probe: {error!r}")
+        await asyncio.sleep(2)
+        leftovers = []
+        for pid, created in owned:
+            try:
+                p = psutil.Process(pid)
+                if p.create_time() == created and p.is_running():
+                    leftovers.append({"pid": pid, "created": created, "status": p.status()})
+                    # Contain an isolated diagnostic's leak; report it, do not call cleanup a pass.
+                    p.kill()
+            except psutil.NoSuchProcess:
+                pass
+        report["owned_processes_requiring_harness_cleanup"] = leftovers
+        report["end_unix"] = time.time()
+        save(args.out / "report.json", report)
+    return report["status"] == "completed"
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--model", required=True)
+    p.add_argument("--out", type=Path, required=True)
+    args = p.parse_args()
+    os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+    from vllm_omni.windows.aio import install_selector_policy
+
+    install_selector_policy()
+    raise SystemExit(0 if asyncio.run(run(args)) else 1)

benchmarks/edge_harness/check_tts_reliability.py

@@ -0,0 +1,172 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Real-weight TTS slow consumer, abort/recovery and owned vocoder-process failure."""
+
+import argparse
+import asyncio
+import os
+import sys
+import time
+import traceback
+from pathlib import Path
+
+from profile_local_text import save
+
+
+async def run(args):
+    import psutil
+    import torch
+    from transformers import AutoTokenizer
+
+    from vllm_omni import AsyncOmni
+    from vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts import Qwen3TTSConfig
+    from vllm_omni.model_executor.models.qwen3_tts.prompt_embeds_builder import Qwen3TTSPromptEmbedsBuilder
+
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tts"))
+    from stream_latency_bench import _cat_audio
+
+    args.out.mkdir(parents=True, exist_ok=False)
+    report = {
+        "status": "running",
+        "argv": sys.argv,
+        "start_unix": time.time(),
+        "scope": (
+            "Short real audio delivery, slow consumer, abort fence/recovery, owned vocoder termination. "
+            "Not audio quality or 64 MiB buffer saturation."
+        ),
+    }
+    omni, owned = None, []
+    save(args.out / "report.json", report)
+    try:
+        cfg = Qwen3TTSConfig.from_pretrained(args.model)
+        tok = AutoTokenizer.from_pretrained(args.model)
+
+        def prompt(text):
+            info = {"task_type": ["CustomVoice"], "text": [text], "language": ["English"], "speaker": ["Vivian"]}
+            length = Qwen3TTSPromptEmbedsBuilder.estimate_prompt_len_from_additional_information(
+                additional_information=info,
+                task_type="CustomVoice",
+                tokenize_prompt=lambda x: tok(x, padding=False)["input_ids"],
+                codec_language_id=cfg.talker_config.codec_language_id,
+                spk_is_dialect=cfg.talker_config.spk_is_dialect,
+            )
+            return {"prompt_token_ids": [0] * length, "additional_information": info}
+
+        short = prompt("The garden is quiet. We can hear the birds singing.")
+        long = prompt("This longer sentence keeps the speech model generating while we test interruption. " * 12)
+        omni = AsyncOmni(model=args.model, deploy_profile="edge", stage_init_timeout=180)
+        params = omni.resolve_sampling_params_list(None, allow_delta_coercion=True)
+
+        def event(output, rid):
+            assert output.request_id == rid, "Unexpected request ID in audio stream"
+            mm = output.outputs[0].multimodal_output if output.outputs else None
+            audio = _cat_audio(mm.get("audio") if mm else None)
+            return {
+                "request_id": output.request_id,
+                "finished": output.finished,
+                "samples": int(audio.numel()) if audio is not None else 0,
+                "finite": bool(torch.isfinite(audio).all()) if audio is not None else True,
+            }
+
+        async def first_audio(stream, rid):
+            async for output in stream:
+                row = event(output, rid)
+                if row["samples"]:
+                    assert row["finite"], "Nonfinite audio before fault probe"
+                    return row
+            raise RuntimeError("No audio before fault probe")
+
+        async def collect(stream, rid, pause=False):
+            rows = []
+            async for output in stream:
+                rows.append(event(output, rid))
+                if pause and len(rows) == 1:
+                    await asyncio.sleep(3)
+            return rows
+
+        for rid, slow in [("baseline", False), ("slow-consumer", True)]:
+            rows = await collect(omni.generate(short, request_id=rid, sampling_params_list=params), rid, slow)
+            report[rid] = rows
+            assert any(r["finished"] for r in rows) and any(r["samples"] for r in rows)
+            assert all(r["finite"] for r in rows)
+        rid = "abort-probe"
+        stream = omni.generate(long, request_id=rid, sampling_params_list=params)
+        first = await asyncio.wait_for(first_audio(stream, rid), 60)
+        start = time.perf_counter()
+        await omni.abort(rid)
+        report["abort"] = {"first": first, "ack_s": time.perf_counter() - start}
+        try:
+            late = await asyncio.wait_for(collect(stream, rid), 10)
+            report["abort"]["late_events"] = late
+            report["abort"]["late_audio_samples"] = sum(r["samples"] for r in late)
+        except asyncio.CancelledError:
+            report["abort"]["terminal"] = "CancelledError"
+        except Exception as error:
+            report["abort"]["terminal_error"] = repr(error)
+        rid = "after-abort"
+        rows = await collect(omni.generate(short, request_id=rid, sampling_params_list=params), rid)
+        report[rid] = rows
+        assert any(r["finished"] for r in rows) and any(r["samples"] for r in rows)
+        save(args.out / "report.json", report)
+        descendants = {p.pid: p for p in psutil.Process().children(recursive=True)}
+        core_handles = [
+            p for client in omni.engine.stage_clients for p in client.resources.engine_manager.processes if p.is_alive()
+        ]
+        assert core_handles and all(p.pid in descendants for p in core_handles)
+        manager_tree = {}
+        for handle in core_handles:
+            core = descendants[handle.pid]
+            manager_tree[core.pid] = core
+            manager_tree.update({p.pid: p for p in core.children(recursive=True)})
+        owned = [(p.pid, p.create_time()) for p in manager_tree.values()]
+        target = omni.engine.stage_clients[-1].resources.engine_manager.processes[0]
+        rid = "vocoder-crash"
+        stream = omni.generate(long, request_id=rid, sampling_params_list=params)
+        report["crash_first"] = await asyncio.wait_for(first_audio(stream, rid), 60)
+        report["crash"] = {"pid": target.pid, "name": target.name, "created": descendants[target.pid].create_time()}
+        start = time.perf_counter()
+        target.terminate()
+        try:
+            report["crash"]["events"] = await asyncio.wait_for(collect(stream, rid), 30)
+            report["crash"]["explicit_error_reported"] = False
+        except asyncio.TimeoutError:
+            report["crash"]["timeout"] = True
+        except (Exception, asyncio.CancelledError) as error:
+            report["crash"].update(explicit_error_reported=True, error=repr(error))
+        report["crash"]["observation_s"] = time.perf_counter() - start
+        report["status"] = "completed"
+    except Exception as error:
+        report.update(status="failed", error=repr(error), traceback=traceback.format_exc())
+        traceback.print_exc()
+    finally:
+        if omni is not None:
+            try:
+                omni.shutdown()
+            except (Exception, asyncio.CancelledError) as error:
+                report.update(status="failed", cleanup_error=repr(error))
+                report.setdefault("error", f"Engine shutdown failed after probe: {error!r}")
+        await asyncio.sleep(2)
+        leftovers = []
+        for pid, created in owned:
+            try:
+                p = psutil.Process(pid)
+                if p.create_time() == created and p.is_running():
+                    leftovers.append({"pid": pid, "created": created, "status": p.status()})
+                    p.kill()
+            except psutil.NoSuchProcess:
+                pass
+        report["owned_processes_requiring_harness_cleanup"] = leftovers
+        report["end_unix"] = time.time()
+        save(args.out / "report.json", report)
+    return report["status"] == "completed"
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--model", required=True)
+    p.add_argument("--out", type=Path, required=True)
+    args = p.parse_args()
+    os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+    from vllm_omni.windows.aio import install_selector_policy
+
+    install_selector_policy()
+    raise SystemExit(0 if asyncio.run(run(args)) else 1)

benchmarks/edge_harness/download_pending_weights.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Resume the missing E2E checkpoints and verify every safetensors SHA-256.
+
+Weights and status files stay outside the Git checkout. This helper can wait for
+an already-running Hugging Face transfer before resuming it, so an interrupted
+interactive session does not leave an unverified partial checkpoint behind.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from huggingface_hub import HfApi, snapshot_download
+
+
+@dataclass(frozen=True)
+class Checkpoint:
+    repo: str
+    revision: str
+    directory: str
+
+
+CHECKPOINTS = {
+    "minicpmo": Checkpoint(
+        "openbmb/MiniCPM-o-4_5",
+        "503e754207c94da6bb26850b4469f367c9ea3582",
+        "MiniCPM-o-4_5",
+    ),
+    "internvla_base": Checkpoint(
+        "InternRobotics/InternVLA-A1-3B",
+        "08cf716d413aa8208a34f346cd8824e6ff901205",
+        "InternVLA-A1-3B",
+    ),
+    "internvla_place_markpen": Checkpoint(
+        "Jia-Zeng/InternVLA-A1-3B-FineTuned-Place_Markpen",
+        "0a557c6a503e6545b369bff2f34c76a410bbd190",
+        "InternVLA-A1-3B-FT-Place_Markpen",
+    ),
+}
+
+
+def _status(path: Path, **fields: object) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(".json.tmp")
+    temporary.write_text(json.dumps(fields, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    temporary.replace(path)
+
+
+def _wait_for_existing(pid: int | None, repo: str) -> None:
+    if pid is None:
+        return
+    command_path = Path(f"/proc/{pid}/cmdline")
+    while command_path.exists():
+        try:
+            command = command_path.read_bytes()
+        except (FileNotFoundError, PermissionError):
+            break
+        if repo.encode() not in command:
+            break
+        time.sleep(30)
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _verify(checkpoint: Checkpoint, directory: Path) -> list[dict[str, object]]:
+    info = HfApi().model_info(checkpoint.repo, revision=checkpoint.revision, files_metadata=True)
+    weights = [sibling for sibling in info.siblings if sibling.rfilename.endswith(".safetensors")]
+    if not weights:
+        raise RuntimeError(f"No safetensors weight files advertised by {checkpoint.repo}")
+    verified = []
+    for sibling in weights:
+        path = directory / sibling.rfilename
+        expected_size = sibling.size
+        expected_hash = sibling.lfs.sha256 if sibling.lfs else None
+        if expected_size is None or expected_hash is None:
+            raise RuntimeError(f"Missing size or SHA-256 metadata for {sibling.rfilename}")
+        if not path.is_file() or path.stat().st_size != expected_size:
+            raise RuntimeError(f"Missing or incomplete weight: {path}")
+        actual_hash = _sha256(path)
+        if actual_hash != expected_hash:
+            raise RuntimeError(f"SHA-256 mismatch for {path}: {actual_hash} != {expected_hash}")
+        verified.append({"file": sibling.rfilename, "bytes": expected_size, "sha256": actual_hash})
+        print(f"Verified {path.name}: {expected_size} bytes", flush=True)
+    return verified
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("model", choices=CHECKPOINTS)
+    parser.add_argument("--model-root", type=Path, required=True)
+    parser.add_argument("--wait-pid", type=int)
+    parser.add_argument("--detach", action="store_true")
+    args = parser.parse_args()
+
+    checkpoint = CHECKPOINTS[args.model]
+    root = args.model_root.resolve()
+    status_path = root / ".weight_download_status" / f"{args.model}.json"
+    if args.detach:
+        log_path = status_path.with_suffix(".log")
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        command = [sys.executable, str(Path(__file__).resolve()), args.model, "--model-root", str(root)]
+        if args.wait_pid is not None:
+            command.extend(["--wait-pid", str(args.wait_pid)])
+        with log_path.open("ab") as log:
+            process = subprocess.Popen(  # noqa: S603
+                command,
+                stdin=subprocess.DEVNULL,
+                stdout=log,
+                stderr=subprocess.STDOUT,
+                start_new_session=True,
+                close_fds=True,
+                env={**os.environ, "HF_XET_HIGH_PERFORMANCE": "1"},
+            )
+        print(json.dumps({"pid": process.pid, "log": str(log_path)}))
+        return
+
+    directory = root / checkpoint.directory
+    _status(status_path, status="waiting", repo=checkpoint.repo, revision=checkpoint.revision)
+    try:
+        _wait_for_existing(args.wait_pid, checkpoint.repo)
+        _status(status_path, status="downloading", repo=checkpoint.repo, revision=checkpoint.revision)
+        snapshot_download(
+            repo_id=checkpoint.repo,
+            revision=checkpoint.revision,
+            local_dir=str(directory),
+            max_workers=4,
+        )
+        _status(status_path, status="verifying", repo=checkpoint.repo, revision=checkpoint.revision)
+        files = _verify(checkpoint, directory)
+        _status(status_path, status="verified", repo=checkpoint.repo, revision=checkpoint.revision, files=files)
+    except Exception as exc:
+        _status(status_path, status="failed", repo=checkpoint.repo, revision=checkpoint.revision, error=str(exc))
+        raise
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/experiments/README.md

@@ -0,0 +1,20 @@
+# One-off experiment drivers
+
+Scripts that produced a specific result rather than tooling meant to be reused.
+Kept because the results in `../results/` are only checkable if the script that
+made them is here.
+
+| group | what |
+|---|---|
+| `aihub_*.py` | Qualcomm AI Hub on-device runs — Snapdragon layer timings, calibration, w8a16/w4a16 parity. Authentication comes from `~/.qai_hub/client.ini`; no credential is in these files |
+| `bench_int4_kernels.py` | the 4-bit GEMM microbenchmark — kernel-level shapes, used to size decisions before building anything |
+| `quality_int4.py`, `ppl_vllm.py`, `parity_hf_vs_vllm.py`, `ref_hf.py` | fidelity: greedy divergence, perplexity, and parity against a HuggingFace reference |
+| `quantize_int8.py`, `export_gptq_cpu.py`, `dequantize_int4.py`, `verify_export.py` | building and round-tripping checkpoints |
+| `capture_real_acts.py`, `capture_lm_head.py` | activation capture for calibrated quantization |
+| `cpu_busy.py` | a load generator, used to test that the harness's contention guard actually refuses |
+
+**They are not portable as written.** Most carry absolute paths from the machine
+they ran on (`/data/<user>/embedding_infer/...`) and expect checkpoints under
+`models/`. They are a record of what was run, not a supported interface — the
+supported entry points are the tools one level up, described in
+[docs/edge/harness.md](../../../docs/edge/harness.md).

benchmarks/edge_harness/experiments/aihub_block4.py

@@ -0,0 +1,50 @@
+"""The architecture's repeating 4-layer block (3 sliding + 1 full) on a real S25.
+
+Profiling one layer at a time pays the graph's Input/Output boundary once per
+layer. A real decode step pays it once for all 28. Measuring 1 layer and 4
+layers at the same context separates the fixed per-graph cost from the true
+per-layer cost, so the 28-layer step can be extrapolated honestly.
+"""
+import json
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+OPT = {
+    "fp16": "--target_runtime qnn_context_binary --quantize_full_type float16",
+    "w4a16": "--target_runtime qnn_context_binary --quantize_full_type w4a16 --quantize_io",
+}
+
+res, pend = {}, {}
+mdl = hub.upload_model(f"{HERE}/onnx/ring_block4_ctx1024.onnx", name="spark25_block4")
+print("uploaded", flush=True)
+for q in ("w4a16", "fp16"):
+    try:
+        cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                                    name=f"spark block4 {q}", options=OPT[q])
+        pend[q] = cj
+        print("compile", q, cj.job_id, flush=True)
+    except Exception as e:
+        print("submit failed", q, str(e)[:200], flush=True)
+
+prof = {}
+for q, cj in pend.items():
+    st = cj.wait()
+    print("compile", q, st.code, (st.message or "")[:200], flush=True)
+    if st.success:
+        prof[q] = hub.submit_profile_job(model=cj.get_target_model(),
+                                         device=hub.Device(DEV),
+                                         name=f"spark block4 {q}",
+                                         options="--compute_unit npu")
+for q, pj in prof.items():
+    st = pj.wait()
+    if st.success:
+        p = pj.download_profile()
+        res[q] = p["execution_summary"]["estimated_inference_time"] / 1e3
+        json.dump(p, open(f"{HERE}/profile_block4_{q}.json", "w"))
+        print(f"  block4 {q}: {res[q]:.3f} ms", flush=True)
+    else:
+        res[q] = None
+        print(f"  block4 {q} FAILED: {(st.message or '')[:250]}", flush=True)
+    json.dump(res, open(f"{HERE}/aihub_block4.json", "w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_calib_any.py

@@ -0,0 +1,78 @@
+"""Calibrated quantisation + on-device parity for any exported graph."""
+import argparse, json
+import numpy as np
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+SCHEMES = {
+    "w8a16": (hub.QuantizeDtype.INT8, hub.QuantizeDtype.INT16),
+    "w16a16": (hub.QuantizeDtype.INT16, hub.QuantizeDtype.INT16),
+}
+
+
+def snr(a, b):
+    a, b = np.asarray(a).ravel(), np.asarray(b).ravel()
+    return float(20 * np.log10(np.linalg.norm(a) / max(np.linalg.norm(a - b), 1e-12)))
+
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--onnx", required=True)
+ap.add_argument("--acts", required=True)
+ap.add_argument("--map", default="", help="onnx_name=npz_name,...")
+ap.add_argument("--tag", required=True)
+ap.add_argument("--out", required=True)
+a = ap.parse_args()
+
+import onnx
+g = onnx.load(a.onnx, load_external_data=False)
+live = [t.name for t in g.graph.input]
+mapping = dict(p.split("=") for p in a.map.split(",") if p)
+d = np.load(a.acts)
+n = d["x"].shape[0]
+calib = {name: [d[mapping.get(name, name)][i] for i in range(n)] for name in live}
+probe = {name: [d[mapping.get(name, name)][0]] for name in live}
+ref_out = d["hidden_out"][0]
+
+src = hub.upload_model(a.onnx, name=f"spark25_{a.tag}_src")
+qj = {}
+for tag, (w, act) in SCHEMES.items():
+    qj[tag] = hub.submit_quantize_job(model=src, calibration_data=calib,
+                                      weights_dtype=w, activations_dtype=act,
+                                      name=f"spark {a.tag} {tag}")
+    print("quantize", tag, qj[tag].job_id, flush=True)
+
+res, cj = {}, {}
+for tag, j in qj.items():
+    st = j.wait()
+    print("quantize", tag, st.code, (st.message or "")[:140], flush=True)
+    if st.success:
+        cj[tag] = hub.submit_compile_job(
+            model=j.get_target_model(), device=hub.Device(DEV),
+            name=f"spark {a.tag} {tag}",
+            options="--target_runtime qnn_context_binary --quantize_io")
+ij, pj = {}, {}
+for tag, j in cj.items():
+    st = j.wait()
+    print("compile", tag, st.code, (st.message or "")[:140], flush=True)
+    if st.success:
+        t = j.get_target_model()
+        ij[tag] = hub.submit_inference_job(model=t, device=hub.Device(DEV),
+                                           inputs=probe, name=f"spark {a.tag} {tag}")
+        pj[tag] = hub.submit_profile_job(model=t, device=hub.Device(DEV),
+                                         name=f"spark {a.tag} {tag}",
+                                         options="--compute_unit npu")
+for tag in list(ij):
+    e = res.setdefault(tag, {})
+    st = ij[tag].wait()
+    if st.success:
+        out = ij[tag].download_output_data()
+        e["snr_db"] = round(snr(ref_out, np.asarray(list(out.values())[0][0])), 2)
+    else:
+        e["snr_db"] = None
+    st = pj[tag].wait()
+    e["ms"] = (pj[tag].download_profile()["execution_summary"]
+               ["estimated_inference_time"] / 1e3) if st.success else None
+    print(f"  {a.tag} {tag}: {e}", flush=True)
+    json.dump(res, open(a.out, "w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_calibrated.py

@@ -0,0 +1,85 @@
+"""Calibrated quantisation of the sliding layer, judged on real activations.
+
+Uncalibrated w4a16/w4a8 scored -1.6 / -1.28 dB against the reference -- output
+uncorrelated with the truth. This feeds the quantiser real activations
+captured from the reference model on real text, then measures the same way.
+"""
+import json
+import numpy as np
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+NAMES = ["x", "cos_sw", "sin_sw", "mask_sw", "k_cache_0", "v_cache_0"]
+
+d = np.load(f"{HERE}/ring3_real_acts.npz")
+n_samples = d["x"].shape[0]
+calib = {n: [d[n][i] for i in range(n_samples)] for n in NAMES}
+probe = {n: [d[n][0]] for n in NAMES}
+ref_out = d["hidden_out"][0]
+
+SCHEMES = {
+    "w4a16": (hub.QuantizeDtype.INT4, hub.QuantizeDtype.INT16),
+    "w8a16": (hub.QuantizeDtype.INT8, hub.QuantizeDtype.INT16),
+    "w4a8": (hub.QuantizeDtype.INT4, hub.QuantizeDtype.INT8),
+    "w8a8": (hub.QuantizeDtype.INT8, hub.QuantizeDtype.INT8),
+}
+
+
+def snr(a, b):
+    a, b = np.asarray(a).ravel(), np.asarray(b).ravel()
+    return float(20 * np.log10(np.linalg.norm(a) / max(np.linalg.norm(a - b), 1e-12)))
+
+
+src = hub.upload_model(f"{HERE}/onnx/ring3_sliding.onnx", name="spark25_calib_src")
+qjobs = {}
+for tag, (w, a) in SCHEMES.items():
+    qj = hub.submit_quantize_job(model=src, calibration_data=calib,
+                                 weights_dtype=w, activations_dtype=a,
+                                 name=f"spark calib {tag}")
+    qjobs[tag] = qj
+    print("quantize", tag, qj.job_id, flush=True)
+
+res = {}
+cjobs = {}
+for tag, qj in qjobs.items():
+    st = qj.wait()
+    print("quantize", tag, st.code, (st.message or "")[:150], flush=True)
+    if st.success:
+        cjobs[tag] = hub.submit_compile_job(
+            model=qj.get_target_model(), device=hub.Device(DEV),
+            name=f"spark calib {tag}",
+            options="--target_runtime qnn_context_binary --quantize_io")
+
+ijobs, pjobs = {}, {}
+for tag, cj in cjobs.items():
+    st = cj.wait()
+    print("compile", tag, st.code, (st.message or "")[:150], flush=True)
+    if st.success:
+        tgt = cj.get_target_model()
+        ijobs[tag] = hub.submit_inference_job(model=tgt, device=hub.Device(DEV),
+                                              inputs=probe, name=f"spark calib {tag}")
+        pjobs[tag] = hub.submit_profile_job(model=tgt, device=hub.Device(DEV),
+                                            name=f"spark calib {tag}",
+                                            options="--compute_unit npu")
+
+for tag in list(ijobs):
+    st = ijobs[tag].wait()
+    entry = res.setdefault(tag, {})
+    if st.success:
+        out = ijobs[tag].download_output_data()
+        got = np.asarray(list(out.values())[0][0])
+        entry["snr_db"] = round(snr(ref_out, got), 2)
+    else:
+        entry["snr_db"] = None
+        print(tag, "inference FAILED", (st.message or "")[:150], flush=True)
+    st = pjobs[tag].wait()
+    if st.success:
+        p = pjobs[tag].download_profile()
+        entry["ms"] = p["execution_summary"]["estimated_inference_time"] / 1e3
+        json.dump(p, open(f"{HERE}/profile_calib_{tag}.json", "w"))
+    else:
+        entry["ms"] = None
+    print(f"  {tag}: {entry}", flush=True)
+    json.dump(res, open(f"{HERE}/aihub_calibrated.json", "w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_cpu_int8.py

@@ -0,0 +1,40 @@
+"""A quantized phone-CPU reference for the same layer.
+
+llama.cpp cannot be run on an AI Hub device, and it would use 4-bit weights on
+the CPU, so comparing our NPU path against an fp16 CPU run would flatter us.
+This asks for the closest thing AI Hub offers: the same graph, int8, on the
+S25's CPU.
+"""
+import json
+import qai_hub as hub
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+CAND = [
+    ("tflite_int8_cpu", "--target_runtime tflite --quantize_full_type int8 --quantize_io", "cpu"),
+    ("tflite_w8a8_cpu", "--target_runtime tflite --quantize_full_type w8a8 --quantize_io", "cpu"),
+]
+res, pend = {}, {}
+mdl = hub.upload_model(f"{HERE}/onnx/ring3_sliding.onnx", name="spark25_ring3_cpu")
+for name, opt, unit in CAND:
+    try:
+        cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                                    name=f"spark {name}", options=opt)
+        pend[(name, unit)] = cj; print("compile", name, cj.job_id, flush=True)
+    except Exception as e:
+        print("submit failed", name, str(e)[:180], flush=True)
+prof = {}
+for key, cj in pend.items():
+    st = cj.wait(); print("compile", key[0], st.code, (st.message or "")[:180], flush=True)
+    if st.success:
+        prof[key] = hub.submit_profile_job(model=cj.get_target_model(), device=hub.Device(DEV),
+                                           name=f"spark {key[0]}", options=f"--compute_unit {key[1]}")
+for key, pj in prof.items():
+    st = pj.wait()
+    if st.success:
+        p = pj.download_profile()
+        res[key[0]] = p["execution_summary"]["estimated_inference_time"]/1e3
+        print(f"  {key[0]}: {res[key[0]]:.3f} ms", flush=True)
+    else:
+        res[key[0]] = None; print(f"  {key[0]} FAILED:", (st.message or "")[:180], flush=True)
+    json.dump(res, open(f"{HERE}/aihub_cpu_int8.json","w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_gelu.py

@@ -0,0 +1,31 @@
+"""Is the cheaper GELU actually cheaper on the Hexagon NPU?
+
+The erf form is 11-13% of every layer in the device profile. tanh holds 65.2 dB
+against the reference with top-1/top-5 intact, so it is adoptable if it buys
+time. This measures whether it does.
+"""
+import json
+import qai_hub as hub
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+OPT = "--target_runtime qnn_context_binary --quantize_full_type float16"
+res = {}
+mdl = hub.upload_model(f"{HERE}/onnx/ring_sliding_tanh.onnx", name="spark25_sliding_tanh")
+cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                            name="spark sliding ring tanh fp16", options=OPT)
+print("compile", cj.job_id, flush=True)
+st = cj.wait(); print("compile", st.code, (st.message or "")[:160], flush=True)
+if st.success:
+    pj = hub.submit_profile_job(model=cj.get_target_model(), device=hub.Device(DEV),
+                                name="spark sliding ring tanh fp16",
+                                options="--compute_unit npu")
+    st2 = pj.wait()
+    if st2.success:
+        p = pj.download_profile()
+        res["ring_sliding_tanh_fp16"] = p["execution_summary"]["estimated_inference_time"] / 1e3
+        json.dump(p, open(f"{HERE}/profile_ring_sliding_tanh_fp16.json", "w"))
+        print(f"  tanh: {res['ring_sliding_tanh_fp16']:.3f} ms", flush=True)
+    else:
+        print("  profile FAILED:", (st2.message or "")[:200], flush=True)
+json.dump(res, open(f"{HERE}/aihub_gelu.json", "w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_int4.py

@@ -0,0 +1,86 @@
+"""int4 weights on the NPU, calibrated.
+
+int4 halves the weight traffic that dominates every matmul in the step, so it
+is the largest remaining lever. The compile-path form (--quantize_full_type
+w4a16) runs at 0.838 ms but is uncalibrated and scores -1.6 dB. The
+quantize-job form carries calibration but failed to compile. This retries that
+compile without --quantize_io, which is the one thing the failing jobs had in
+common with an already-quantized input model.
+"""
+import argparse, json
+import numpy as np
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--onnx", default=f"{HERE}/onnx/ring3_sliding.onnx")
+ap.add_argument("--acts", default=f"{HERE}/ring3_real_acts.npz")
+ap.add_argument("--map", default="")
+ap.add_argument("--tag", default="sliding")
+ap.add_argument("--out", default=f"{HERE}/aihub_int4.json")
+a = ap.parse_args()
+
+import onnx
+live = [t.name for t in onnx.load(a.onnx, load_external_data=False).graph.input]
+mapping = dict(p.split("=") for p in a.map.split(",") if p)
+d = np.load(a.acts)
+n = d["x"].shape[0]
+calib = {x: [d[mapping.get(x, x)][i] for i in range(n)] for x in live}
+probe = {x: [d[mapping.get(x, x)][0]] for x in live}
+ref = d["hidden_out"][0]
+
+
+def snr(u, v):
+    u, v = np.asarray(u).ravel(), np.asarray(v).ravel()
+    return float(20 * np.log10(np.linalg.norm(u) / max(np.linalg.norm(u - v), 1e-12)))
+
+
+src = hub.upload_model(a.onnx, name=f"spark25_{a.tag}_int4")
+qj = hub.submit_quantize_job(model=src, calibration_data=calib,
+                             weights_dtype=hub.QuantizeDtype.INT4,
+                             activations_dtype=hub.QuantizeDtype.INT16,
+                             name=f"spark {a.tag} int4 calib")
+print("quantize", qj.job_id, flush=True)
+st = qj.wait()
+print("quantize", st.code, (st.message or "")[:200], flush=True)
+res = {}
+if st.success:
+    qm = qj.get_target_model()
+    # The failing attempts all passed --quantize_io on an already-quantized
+    # model; try without it, and with the plain QNN DLC target as a fallback.
+    VARIANTS = {
+        "ctx_no_qio": "--target_runtime qnn_context_binary",
+        "ctx_qio": "--target_runtime qnn_context_binary --quantize_io",
+    }
+    cj = {}
+    for tag, opt in VARIANTS.items():
+        try:
+            cj[tag] = hub.submit_compile_job(model=qm, device=hub.Device(DEV),
+                                             name=f"spark {a.tag} int4 {tag}",
+                                             options=opt)
+            print("compile", tag, cj[tag].job_id, flush=True)
+        except Exception as e:
+            print("submit failed", tag, str(e)[:160], flush=True)
+    for tag, j in cj.items():
+        s2 = j.wait()
+        print("compile", tag, s2.code, (s2.message or "")[:200], flush=True)
+        e = res.setdefault(tag, {})
+        if not s2.success:
+            e["ms"] = e["snr_db"] = None
+            json.dump(res, open(a.out, "w"), indent=1)
+            continue
+        t = j.get_target_model()
+        ij = hub.submit_inference_job(model=t, device=hub.Device(DEV), inputs=probe,
+                                      name=f"spark {a.tag} int4 {tag}")
+        pj = hub.submit_profile_job(model=t, device=hub.Device(DEV),
+                                    name=f"spark {a.tag} int4 {tag}",
+                                    options="--compute_unit npu")
+        s3 = ij.wait()
+        e["snr_db"] = round(snr(ref, np.asarray(list(ij.download_output_data().values())[0][0])), 2) if s3.success else None
+        s4 = pj.wait()
+        e["ms"] = pj.download_profile()["execution_summary"]["estimated_inference_time"] / 1e3 if s4.success else None
+        print(f"  {a.tag} int4 {tag}: {e}", flush=True)
+        json.dump(res, open(a.out, "w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_parity_a8.py

@@ -0,0 +1,60 @@
+"""On-device fidelity of the quantized sliding layer.
+
+Speed is only half the answer: this repo has already found automatic int8
+numerically broken on Qwen3-TTS and MiniCPM-o. This runs the real layer on a
+real S25 at each quantisation and compares against the fp32 reference.
+"""
+import json
+import numpy as np
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+OPT = {
+    "fp16": "--target_runtime qnn_context_binary --quantize_full_type float16",
+    "w4a16": "--target_runtime qnn_context_binary --quantize_full_type w4a16 --quantize_io",
+    "w4a8": "--target_runtime qnn_context_binary --quantize_full_type w4a8 --quantize_io",
+}
+
+ref = np.load(f"{HERE}/ring3_ref_inputs.npz")
+# The graph's live inputs; ONNX prunes the full-attention rotary/mask tensors
+# a sliding layer never reads.
+names = ["x", "cos_sw", "sin_sw", "mask_sw", "k_cache_0", "v_cache_0"]
+inputs = {n: [ref[n].astype(np.float32)] for n in names}
+ref_out = ref["hidden_out"]
+
+
+def snr(a, b):
+    a, b = a.ravel(), b.ravel()
+    return float(20 * np.log10(np.linalg.norm(a) / max(np.linalg.norm(a - b), 1e-12)))
+
+
+mdl = hub.upload_model(f"{HERE}/onnx/ring3_sliding.onnx", name="spark25_ring3_parity")
+res, pend = {}, {}
+for q, o in OPT.items():
+    cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                                name=f"spark parity {q}", options=o)
+    pend[q] = cj
+    print("compile", q, cj.job_id, flush=True)
+infj = {}
+for q, cj in pend.items():
+    st = cj.wait()
+    if st.success:
+        infj[q] = hub.submit_inference_job(model=cj.get_target_model(),
+                                           device=hub.Device(DEV), inputs=inputs,
+                                           name=f"spark parity {q}")
+    else:
+        print("compile", q, "FAILED", (st.message or "")[:160], flush=True)
+for q, ij in infj.items():
+    st = ij.wait()
+    if st.success:
+        out = ij.download_output_data()
+        key = [k for k in out if "hidden" in k or "output" in k.lower()] or list(out)
+        got = np.asarray(out[key[0]][0])
+        res[q] = round(snr(ref_out, got), 2)
+        print(f"  {q}: {res[q]} dB", flush=True)
+    else:
+        res[q] = None
+        print(f"  {q} inference FAILED:", (st.message or "")[:160], flush=True)
+    json.dump(res, open(f"{HERE}/aihub_parity_a8.json", "w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_processors.py

@@ -0,0 +1,54 @@
+"""The ring decode layers across every processor the S25 exposes.
+
+llama.cpp cannot be run on an AI Hub device, so the honest phone-CPU
+reference is the same graph compiled for TFLite and run on the CPU. That
+gives an NPU-vs-CPU ratio on identical work, on identical silicon.
+"""
+import json
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+GRAPHS = ["ring_sliding_ctx1024", "ring_full_ctx1024"]
+TARGETS = [
+    ("qnn_npu", "--target_runtime qnn_context_binary --quantize_full_type float16", "npu"),
+    ("tflite_npu", "--target_runtime tflite", "npu"),
+    ("tflite_cpu", "--target_runtime tflite", "cpu"),
+    ("tflite_gpu", "--target_runtime tflite", "gpu"),
+    ("onnx_npu", "--target_runtime onnx", "npu"),
+]
+
+pend, res = {}, {}
+for g in GRAPHS:
+    mdl = hub.upload_model(f"{HERE}/onnx/{g}.onnx", name=f"spark25_{g}_proc")
+    for name, opt, unit in TARGETS:
+        try:
+            cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                                        name=f"spark {g} {name}", options=opt)
+            pend[(g, name, unit)] = cj
+            print("compile", g, name, cj.job_id, flush=True)
+        except Exception as e:
+            print("submit failed", g, name, str(e)[:140], flush=True)
+
+prof = {}
+for key, cj in pend.items():
+    st = cj.wait()
+    if st.success:
+        prof[key] = hub.submit_profile_job(model=cj.get_target_model(),
+                                           device=hub.Device(DEV),
+                                           name=f"spark {key[0]} {key[1]}",
+                                           options=f"--compute_unit {key[2]}")
+    else:
+        print("compile", key, "FAILED", (st.message or "")[:140], flush=True)
+for key, pj in prof.items():
+    st = pj.wait()
+    tag = f"{key[0]}|{key[1]}"
+    if st.success:
+        p = pj.download_profile()
+        res[tag] = p["execution_summary"]["estimated_inference_time"] / 1e3
+        print(f"  {tag}: {res[tag]:.3f} ms", flush=True)
+    else:
+        res[tag] = None
+        print(f"  {tag} FAILED: {(st.message or '')[:160]}", flush=True)
+    json.dump(res, open(f"{HERE}/aihub_processors.json", "w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_ring.py

@@ -0,0 +1,64 @@
+"""Ring-buffer decode step on a real Galaxy S25, plus the output head.
+
+The roll layout spent ~20% of a sliding layer writing the recomputed window
+back out. The ring layout returns one K/V entry instead, so this measures
+what that bought. The output head (2048 x 131072, tied to the embedding) was
+missing from the earlier per-layer numbers entirely and is measured here.
+"""
+import json
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+
+JOBS = [
+    ("ring_sliding_ctx1024", ["fp16", "w4a16"]),
+    ("ring_full_ctx1024", ["fp16", "w4a16"]),
+    ("ring_full_ctx4096", ["w4a16"]),
+    ("lm_head", ["w4a16", "w8a16"]),
+]
+OPT = {
+    "fp16": "--target_runtime qnn_context_binary --quantize_full_type float16",
+    "w8a16": "--target_runtime qnn_context_binary --quantize_full_type w8a16 --quantize_io",
+    "w4a16": "--target_runtime qnn_context_binary --quantize_full_type w4a16 --quantize_io",
+}
+
+
+def main():
+    pend = {}
+    for name, quants in JOBS:
+        mdl = hub.upload_model(f"{HERE}/onnx/{name}.onnx", name=f"spark25_{name}")
+        print("uploaded", name, flush=True)
+        for q in quants:
+            try:
+                cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                                            name=f"spark {name} {q}", options=OPT[q])
+                pend[(name, q)] = cj
+                print("compile", name, q, cj.job_id, flush=True)
+            except Exception as e:
+                print("submit failed", name, q, str(e)[:160], flush=True)
+
+    prof, res = {}, {}
+    for key, cj in pend.items():
+        st = cj.wait()
+        print("compile", key, st.code, (st.message or "")[:150], flush=True)
+        if st.success:
+            prof[key] = hub.submit_profile_job(
+                model=cj.get_target_model(), device=hub.Device(DEV),
+                name=f"spark {key[0]} {key[1]}", options="--compute_unit npu")
+    for key, pj in prof.items():
+        st = pj.wait()
+        tag = f"{key[0]}|{key[1]}"
+        if st.success:
+            p = pj.download_profile()
+            res[tag] = p["execution_summary"]["estimated_inference_time"] / 1e3
+            json.dump(p, open(f"{HERE}/profile_{key[0]}_{key[1]}.json", "w"))
+            print(f"  {tag}: {res[tag]:.3f} ms", flush=True)
+        else:
+            res[tag] = None
+            print(f"  {tag} FAILED: {(st.message or '')[:200]}", flush=True)
+        json.dump(res, open(f"{HERE}/aihub_ring.json", "w"), indent=1)
+    print("done", flush=True)
+
+
+main()

benchmarks/edge_harness/experiments/aihub_ring2.py

@@ -0,0 +1,33 @@
+"""Ring sliding layer, concat-free (online softmax), at w4a16 and fp16."""
+import json
+import qai_hub as hub
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+OPT = {
+    "w4a16": "--target_runtime qnn_context_binary --quantize_full_type w4a16 --quantize_io",
+    "fp16": "--target_runtime qnn_context_binary --quantize_full_type float16",
+}
+mdl = hub.upload_model(f"{HERE}/onnx/ring2_sliding_ctx1024.onnx", name="spark25_ring2_sliding")
+pend, res = {}, {}
+for q, o in OPT.items():
+    cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                                name=f"spark ring2 sliding {q}", options=o)
+    pend[q] = cj; print("compile", q, cj.job_id, flush=True)
+prof = {}
+for q, cj in pend.items():
+    st = cj.wait(); print("compile", q, st.code, (st.message or "")[:180], flush=True)
+    if st.success:
+        prof[q] = hub.submit_profile_job(model=cj.get_target_model(), device=hub.Device(DEV),
+                                         name=f"spark ring2 sliding {q}",
+                                         options="--compute_unit npu")
+for q, pj in prof.items():
+    st = pj.wait()
+    if st.success:
+        p = pj.download_profile()
+        res[q] = p["execution_summary"]["estimated_inference_time"]/1e3
+        json.dump(p, open(f"{HERE}/profile_ring2_sliding_{q}.json","w"))
+        print(f"  ring2 sliding {q}: {res[q]:.3f} ms", flush=True)
+    else:
+        res[q] = None; print(f"  {q} FAILED:", (st.message or "")[:200], flush=True)
+    json.dump(res, open(f"{HERE}/aihub_ring2.json","w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_ring3.py

@@ -0,0 +1,33 @@
+"""Ring sliding layer, concat-free (online softmax), at w4a16 and fp16."""
+import json
+import qai_hub as hub
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+OPT = {
+    "w4a16": "--target_runtime qnn_context_binary --quantize_full_type w4a16 --quantize_io",
+    "fp16": "--target_runtime qnn_context_binary --quantize_full_type float16",
+}
+mdl = hub.upload_model(f"{HERE}/onnx/ring3_sliding.onnx", name="spark25_ring3_sliding")
+pend, res = {}, {}
+for q, o in OPT.items():
+    cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                                name=f"spark ring3 sliding {q}", options=o)
+    pend[q] = cj; print("compile", q, cj.job_id, flush=True)
+prof = {}
+for q, cj in pend.items():
+    st = cj.wait(); print("compile", q, st.code, (st.message or "")[:180], flush=True)
+    if st.success:
+        prof[q] = hub.submit_profile_job(model=cj.get_target_model(), device=hub.Device(DEV),
+                                         name=f"spark ring3 sliding {q}",
+                                         options="--compute_unit npu")
+for q, pj in prof.items():
+    st = pj.wait()
+    if st.success:
+        p = pj.download_profile()
+        res[q] = p["execution_summary"]["estimated_inference_time"]/1e3
+        json.dump(p, open(f"{HERE}/profile_ring3_sliding_{q}.json","w"))
+        print(f"  ring3 sliding {q}: {res[q]:.3f} ms", flush=True)
+    else:
+        res[q] = None; print(f"  {q} FAILED:", (st.message or "")[:200], flush=True)
+    json.dump(res, open(f"{HERE}/aihub_ring3.json","w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_ring_w4.py

@@ -0,0 +1,44 @@
+"""w4a16 retry for the ring layout, after expressing the new-token term as a
+matmul instead of a two-axis broadcast multiply (which QNN's context-binary
+converter rejected at w4a16 while accepting it at fp16)."""
+import json
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+OPT = "--target_runtime qnn_context_binary --quantize_full_type w4a16 --quantize_io"
+NAMES = ["ring_sliding_ctx1024", "ring_full_ctx1024", "ring_full_ctx4096",
+         ]
+
+pend, res = {}, {}
+for n in NAMES:
+    mdl = hub.upload_model(f"{HERE}/onnx/{n}.onnx", name=f"spark25_{n}_v3")
+    try:
+        cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                                    name=f"spark {n} w4a16 v3", options=OPT)
+        pend[n] = cj
+        print("compile", n, cj.job_id, flush=True)
+    except Exception as e:
+        print("submit failed", n, str(e)[:200], flush=True)
+
+prof = {}
+for n, cj in pend.items():
+    st = cj.wait()
+    print("compile", n, st.code, (st.message or "")[:200], flush=True)
+    if st.success:
+        prof[n] = hub.submit_profile_job(model=cj.get_target_model(),
+                                         device=hub.Device(DEV),
+                                         name=f"spark {n} w4a16 v3",
+                                         options="--compute_unit npu")
+for n, pj in prof.items():
+    st = pj.wait()
+    if st.success:
+        p = pj.download_profile()
+        res[n] = p["execution_summary"]["estimated_inference_time"] / 1e3
+        json.dump(p, open(f"{HERE}/profile_{n}_w4a16.json", "w"))
+        print(f"  {n}: {res[n]:.3f} ms", flush=True)
+    else:
+        res[n] = None
+        print(f"  {n} FAILED: {(st.message or '')[:250]}", flush=True)
+    json.dump(res, open(f"{HERE}/aihub_ring_w4_v3.json", "w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_spark_layers.py

@@ -0,0 +1,93 @@
+"""Profile Spark-X2.5 decoder layers on real Snapdragon hardware.
+
+One sliding layer and one full-attention layer are profiled separately. The
+sliding layer's cache is a fixed 512 entries at any context; the full layer's
+grows, so it is profiled at 1024 and 4096 to show the slope. A whole decode
+step is 21 sliding + 7 full layers plus the head.
+"""
+import json, sys
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+HID, HEADS, KV, D = 2048, 8, 2, 256
+SW = 512
+
+GRAPHS = [
+    ("sliding", 1024, SW),
+    ("full", 1024, 1024),
+    ("full", 4096, 4096),
+]
+QUANTS = [
+    ("fp16", "--target_runtime qnn_context_binary --quantize_full_type float16"),
+    ("w8a16", "--target_runtime qnn_context_binary --quantize_full_type w8a16 --quantize_io"),
+    ("w4a16", "--target_runtime qnn_context_binary --quantize_full_type w4a16 --quantize_io"),
+]
+DEVICES = ["Samsung Galaxy S25"]
+
+
+def spec_for(ctx, cache_len):
+    s = {
+        "x": ((1, 1, HID), "float32"),
+        "cos_sw": ((1, 1, 1, D), "float32"),
+        "sin_sw": ((1, 1, 1, D), "float32"),
+        "cos_full": ((1, 1, 1, D // 4), "float32"),
+        "sin_full": ((1, 1, 1, D // 4), "float32"),
+        "mask_sw": ((1, 1, 1, SW), "float32"),
+        "mask_full": ((1, 1, 1, ctx + 1), "float32"),
+        "k_cache_0": ((1, KV, cache_len, D), "float32"),
+        "v_cache_0": ((1, KV, cache_len, D), "float32"),
+    }
+    return s
+
+
+def main():
+    results, pending = {}, {}
+    for kind, ctx, cache_len in GRAPHS:
+        path = f"{HERE}/onnx/spark_{kind}_ctx{ctx}.onnx"
+        mdl = hub.upload_model(path, name=f"spark25_{kind}_ctx{ctx}")
+        print("uploaded", kind, ctx, flush=True)
+        for dev in DEVICES:
+            for q, opt in QUANTS:
+                key = (kind, ctx, dev, q)
+                try:
+                    # No input_specs: every shape is static, and the ONNX
+                    # export prunes the inputs a given layer type never
+                    # reads (a full layer has no sliding mask), so a
+                    # hand-written spec would not match the graph.
+                    cj = hub.submit_compile_job(
+                        model=mdl, device=hub.Device(dev),
+                        name=f"spark {kind} ctx{ctx} {q}", options=opt)
+                    pending[key] = cj
+                    print("compile submitted", key, cj.job_id, flush=True)
+                except Exception as e:
+                    print("submit failed", key, str(e)[:160], flush=True)
+
+    prof = {}
+    for key, cj in pending.items():
+        st = cj.wait()
+        print("compile", key, st.code, (st.message or "")[:160], flush=True)
+        if st.success:
+            prof[key] = hub.submit_profile_job(
+                model=cj.get_target_model(), device=hub.Device(key[2]),
+                name=f"spark {key[0]} ctx{key[1]} {key[3]}",
+                options="--compute_unit npu")
+
+    for key, pj in prof.items():
+        st = pj.wait()
+        kind, ctx, dev, q = key
+        tag = f"{kind}|ctx{ctx}|{dev}|{q}"
+        if st.success:
+            p = pj.download_profile()
+            ms = p["execution_summary"]["estimated_inference_time"] / 1e3
+            results[tag] = ms
+            json.dump(p, open(f"{HERE}/profile_spark_{kind}_{ctx}_{q}.json", "w"))
+            print(f"  {tag}: {ms:.3f} ms/layer", flush=True)
+        else:
+            results[tag] = None
+            print(f"  {tag} FAILED: {(st.message or '')[:200]}", flush=True)
+        json.dump(results, open(f"{HERE}/aihub_spark_layers.json", "w"), indent=1)
+    print("done", flush=True)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

benchmarks/edge_harness/experiments/aihub_spark_runtimes.py

@@ -0,0 +1,59 @@
+"""Same Spark layer, three phone runtimes, one device.
+
+Isolates runtime choice from export choice: if QNN, TFLite and ONNX Runtime
+land close together, the export layout is what sets on-device speed, not the
+engine. Run on the sliding layer, which is 21 of Spark's 28 layers.
+"""
+import json
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+PATH = f"{HERE}/onnx/spark_sliding_ctx1024.onnx"
+
+# Each runtime spells precision differently: only QNN takes
+# --quantize_full_type, TFLite and ONNX reject it outright, and a QNN context
+# binary is NPU-only so the GPU leg has to go through TFLite.
+RUNTIMES = [
+    ("qnn_npu", "--target_runtime qnn_context_binary --quantize_full_type float16", "npu"),
+    ("tflite_npu", "--target_runtime tflite", "npu"),
+    ("onnx_npu", "--target_runtime onnx", "npu"),
+    ("tflite_gpu", "--target_runtime tflite", "gpu"),
+    ("tflite_cpu", "--target_runtime tflite", "cpu"),
+]
+
+
+def main():
+    mdl = hub.upload_model(PATH, name="spark25_sliding_runtimes")
+    pend = {}
+    for name, opt, unit in RUNTIMES:
+        try:
+            cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                                        name=f"spark sliding {name}", options=opt)
+            pend[name] = (cj, unit)
+            print("compile", name, cj.job_id, flush=True)
+        except Exception as e:
+            print("submit failed", name, str(e)[:160], flush=True)
+
+    prof, res = {}, {}
+    for name, (cj, unit) in pend.items():
+        st = cj.wait()
+        print("compile", name, st.code, (st.message or "")[:140], flush=True)
+        if st.success:
+            prof[name] = hub.submit_profile_job(
+                model=cj.get_target_model(), device=hub.Device(DEV),
+                name=f"spark sliding {name}", options=f"--compute_unit {unit}")
+    for name, pj in prof.items():
+        st = pj.wait()
+        if st.success:
+            p = pj.download_profile()
+            res[name] = p["execution_summary"]["estimated_inference_time"] / 1e3
+            print(f"  {name}: {res[name]:.3f} ms/layer", flush=True)
+        else:
+            res[name] = None
+            print(f"  {name} FAILED: {(st.message or '')[:180]}", flush=True)
+        json.dump(res, open(f"{HERE}/aihub_spark_runtimes.json", "w"), indent=1)
+    print("done", flush=True)
+
+
+main()

benchmarks/edge_harness/experiments/aihub_w8a8.py

@@ -0,0 +1,42 @@
+"""int8 activations: does halving KV-cache traffic pay?
+
+Input -- reading the layer's KV cache -- is 13.4% of a sliding layer, and at
+w4a16 the cache crosses the boundary as int16. w8a8 halves that and uses int8
+MACs. Prior work in this repo found automatic int8 numerically broken on two
+other model families, so this measures speed first and only then asks whether
+the fidelity is survivable.
+"""
+import json
+import qai_hub as hub
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+CAND = {
+    "w8a8": "--target_runtime qnn_context_binary --quantize_full_type w8a8 --quantize_io",
+    "w4a8": "--target_runtime qnn_context_binary --quantize_full_type w4a8 --quantize_io",
+}
+mdl = hub.upload_model(f"{HERE}/onnx/ring3_sliding.onnx", name="spark25_ring3_a8")
+pend, res = {}, {}
+for q, o in CAND.items():
+    try:
+        cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                                    name=f"spark ring3 {q}", options=o)
+        pend[q] = cj; print("compile", q, cj.job_id, flush=True)
+    except Exception as e:
+        print("submit failed", q, str(e)[:180], flush=True)
+prof = {}
+for q, cj in pend.items():
+    st = cj.wait(); print("compile", q, st.code, (st.message or "")[:180], flush=True)
+    if st.success:
+        prof[q] = hub.submit_profile_job(model=cj.get_target_model(), device=hub.Device(DEV),
+                                         name=f"spark ring3 {q}", options="--compute_unit npu")
+for q, pj in prof.items():
+    st = pj.wait()
+    if st.success:
+        p = pj.download_profile()
+        res[q] = p["execution_summary"]["estimated_inference_time"]/1e3
+        json.dump(p, open(f"{HERE}/profile_ring3_{q}.json","w"))
+        print(f"  ring3 {q}: {res[q]:.3f} ms", flush=True)
+    else:
+        res[q] = None; print(f"  {q} FAILED:", (st.message or "")[:180], flush=True)
+    json.dump(res, open(f"{HERE}/aihub_w8a8.json","w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/analyze_tts_sustained.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Summarize fixed time windows of raw Qwen3-TTS sustained request streams."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+from pathlib import Path
+
+
+WINDOWS_MIN = ((0, 25), (25, 27), (27, 30))
+
+
+def nearest_rank(values: list[float], fraction: float) -> float | None:
+    if not values:
+        return None
+    ordered = sorted(values)
+    return ordered[math.ceil(len(ordered) * fraction) - 1]
+
+
+def required_prebuffer_ms(row: dict) -> float:
+    chunks = [chunk for chunk in row["chunks_all"] if chunk["samples"]]
+    if not chunks:
+        raise ValueError(f"{row['request_id']}: no audio chunks")
+    first_ms = chunks[0]["t_ms"]
+    played_ms = 0.0
+    required_ms = 0.0
+    for chunk in chunks:
+        required_ms = max(required_ms, chunk["t_ms"] - first_ms - played_ms)
+        played_ms += 1000 * chunk["samples"] / chunk["sr"]
+    return required_ms
+
+
+def summarize(path: Path) -> dict:
+    rows = [json.loads(line) for line in path.open(encoding="utf-8")]
+    sustained = [row for row in rows if row["phase"] == "sustained"]
+    if not sustained:
+        raise ValueError(f"{path}: no sustained requests")
+    start = sustained[0]["submitted_unix"]
+    result = {"source": str(path), "sustained_count": len(sustained), "windows": []}
+    for lo, hi in WINDOWS_MIN:
+        selected = [row for row in sustained
+                    if lo * 60 <= row["submitted_unix"] - start < hi * 60]
+        rtf = [row["rtf_total"] for row in selected]
+        wall = [row["total_wall_s"] for row in selected]
+        buffers = [required_prebuffer_ms(row) for row in selected]
+        result["windows"].append({
+            "minutes": [lo, hi],
+            "requests": len(selected),
+            "rtf_p50": nearest_rank(rtf, 0.5),
+            "rtf_p95": nearest_rank(rtf, 0.95),
+            "wall_s_p50": nearest_rank(wall, 0.5),
+            "wall_s_p95": nearest_rank(wall, 0.95),
+            "requests_with_underrun_at_ttfa": sum(bool(row["underruns"]) for row in selected),
+            "required_prebuffer_ms_p50": nearest_rank(buffers, 0.5),
+            "required_prebuffer_ms_p95": nearest_rank(buffers, 0.95),
+            "required_prebuffer_ms_max": max(buffers) if buffers else None,
+        })
+    return result
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("paths", nargs="+", type=Path)
+    parser.add_argument("--out", type=Path)
+    args = parser.parse_args()
+    result = {
+        "method": "Non-overlapping windows from each run's first sustained submission; nearest-rank percentiles; "
+                  "prebuffer is minimum added delay after first audio arrival to avoid all simulated underruns",
+        "runs": [summarize(path) for path in args.paths],
+    }
+    output = json.dumps(result, indent=2) + "\n"
+    if args.out:
+        args.out.parent.mkdir(parents=True, exist_ok=True)
+        args.out.write_text(output, encoding="utf-8")
+    else:
+        print(output, end="")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/experiments/analyze_tts_telemetry.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Join a complete sustained TTS request run with device-wide GPU telemetry."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import Counter
+from pathlib import Path
+
+from analyze_tts_sustained import WINDOWS_MIN, nearest_rank, summarize
+
+
+METRICS = (
+    "gpu_temperature_c",
+    "gpu_power_mw",
+    "gpu_clock_sm_mhz",
+    "gpu_clock_memory_mhz",
+    "gpu_utilization_pct",
+    "gpu_memory_used_bytes",
+    "host_available_bytes",
+    "host_cpu_utilization_pct",
+)
+
+
+def analyze(report_path: Path, request_path: Path, telemetry_path: Path) -> dict:
+    report = json.loads(report_path.read_text(encoding="utf-8"))
+    if report.get("status") != "completed" or report.get("sustained_wall_s", 0) < 1800:
+        raise ValueError("A terminal 30-minute sustained report is required")
+    if report.get("gpu_telemetry", {}).get("error"):
+        raise ValueError("GPU telemetry reported an error")
+    if report.get("gpu_telemetry", {}).get("samples", 0) < 1500:
+        raise ValueError("Too few device telemetry samples for a 30-minute run")
+    start = report["sustained_start_unix"]
+    base = summarize(request_path)
+    telemetry = [json.loads(line) for line in telemetry_path.open(encoding="utf-8")]
+    if len(telemetry) != report["gpu_telemetry"]["samples"]:
+        raise ValueError("Telemetry sample count differs from the final report")
+    for (lo, hi), window in zip(WINDOWS_MIN, base["windows"], strict=True):
+        samples = [row for row in telemetry if lo * 60 <= row["unix"] - start < hi * 60]
+        window["device_telemetry_samples"] = len(samples)
+        window["device_telemetry"] = {}
+        for name in METRICS:
+            values = [row[name] for row in samples if row.get(name) is not None]
+            window["device_telemetry"][name] = {
+                "p50": nearest_rank(values, 0.5),
+                "p95": nearest_rank(values, 0.95),
+                "min": min(values) if values else None,
+                "max": max(values) if values else None,
+                "available_samples": len(values),
+            }
+        window["gpu_throttle_reason_counts"] = dict(sorted(Counter(
+            str(row["gpu_throttle_reasons"]) for row in samples
+        ).items()))
+    return {
+        "scope": "complete-model Qwen3-TTS stream; simulated playback; telemetry covers whole GPU/host, not just this process",
+        "report": str(report_path),
+        "requests": str(request_path),
+        "telemetry": str(telemetry_path),
+        "loaded_omni_path": report.get("loaded_omni_path"),
+        "cwd": report.get("cwd"),
+        "start_unix": start,
+        "sustained_wall_s": report["sustained_wall_s"],
+        "windows": base["windows"],
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--run-dir", required=True, type=Path)
+    parser.add_argument("--out", type=Path)
+    args = parser.parse_args()
+    result = analyze(
+        args.run_dir / "report.json",
+        args.run_dir / "requests.jsonl",
+        args.run_dir / "gpu_telemetry.jsonl",
+    )
+    output = json.dumps(result, indent=2) + "\n"
+    if args.out:
+        args.out.parent.mkdir(parents=True, exist_ok=True)
+        args.out.write_text(output, encoding="utf-8")
+    else:
+        print(output, end="")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/experiments/audit_internvla_cosmos_qualcomm.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""Audit one hosted InternVLA Cosmos encoder output against a pinned source fixture."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+from collections import Counter
+from pathlib import Path
+
+import numpy as np
+
+
+MODEL_SHA256 = "656a5263f584851a1457ef6dcf60b995f7c67b056b3c90a4f33c814f7caf9ec9"
+FIXTURE_SHA256 = "dcbaf1391ca914d4bbccc1cb384daf05666e771d9cdae90c4f069ec8fcf05368"
+COMPILE_OPTIONS = "--target_runtime qnn_dlc --qnn_options default_graph_htp_precision=FLOAT16"
+
+
+def file_record(path: Path) -> dict:
+    with path.open("rb") as source:
+        digest = hashlib.file_digest(source, "sha256").hexdigest()
+    return {"bytes": path.stat().st_size, "sha256": digest}
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    for name in (
+        "source-onnx", "fixture", "compile-report", "inference-submission",
+        "inference-report", "device-output", "output-report",
+    ):
+        parser.add_argument(f"--{name}", type=Path, required=True)
+    parser.add_argument("--device-name", default="Samsung Galaxy S25")
+    parser.add_argument("--profile-report", type=Path)
+    parser.add_argument("--expected-compile-options", default=COMPILE_OPTIONS)
+    parser.add_argument("--expected-run-options", default="--compute_unit npu")
+    args = parser.parse_args()
+
+    compiled = json.loads(args.compile_report.read_text(encoding="utf-8"))
+    submitted = json.loads(args.inference_submission.read_text(encoding="utf-8"))
+    inferred = json.loads(args.inference_report.read_text(encoding="utf-8"))
+    if compiled.get("status") != "SUCCESS" or inferred.get("status") != "SUCCESS":
+        raise ValueError("compile and inference must be terminal successes")
+    if (compiled.get("source_model_id") != "mn4okj3rq"
+            or compiled.get("options") != args.expected_compile_options):
+        raise ValueError("compile used a different source or options")
+    if inferred.get("model_id") != compiled.get("target_model_id"):
+        raise ValueError("inference used a different target model")
+    if inferred.get("device") != compiled.get("device"):
+        raise ValueError("compile/inference device metadata differs")
+    if inferred.get("device", {}).get("name") != args.device_name:
+        raise ValueError("inference did not use the requested exact device")
+    if inferred.get("options") != args.expected_run_options:
+        raise ValueError("inference did not request the expected compute unit")
+    if (inferred.get("job_id") != submitted.get("job_id")
+            or inferred.get("input_dataset_id") != submitted.get("input_dataset_id")):
+        raise ValueError("inference used a different job or input dataset")
+    if inferred.get("input_dataset_id") != "d26qo8p57":
+        raise ValueError("inference did not use the pinned pattern dataset")
+    if submitted.get("fixture_key", "pattern_pixels") != "pattern_pixels":
+        raise ValueError("submission did not use the pattern fixture")
+
+    files = {
+        name: file_record(path)
+        for name, path in {
+            "source_onnx": args.source_onnx,
+            "fixture": args.fixture,
+            "device_output": args.device_output,
+        }.items()
+    }
+    if files["source_onnx"]["sha256"] != MODEL_SHA256 or files["fixture"]["sha256"] != FIXTURE_SHA256:
+        raise ValueError("source model or fixture hash differs")
+    with np.load(args.fixture, allow_pickle=False) as fixture:
+        if fixture.files != ["ramp_pixels", "ramp_reference", "pattern_pixels", "pattern_reference"]:
+            raise ValueError("fixture layout differs")
+        pixels = np.asarray(fixture["pattern_pixels"])
+        reference = np.asarray(fixture["pattern_reference"])
+        if pixels.shape != (6, 3, 256, 256) or pixels.dtype != np.float32 or not np.isfinite(pixels).all():
+            raise ValueError("input pixels violate fixed source contract")
+        if reference.shape != (6, 16, 32, 32) or reference.dtype != np.float32 or not np.isfinite(reference).all():
+            raise ValueError("source reference violates fixed latent contract")
+    with np.load(args.device_output, allow_pickle=False) as output:
+        if output.files != ["output_0__0"]:
+            raise ValueError("unexpected hosted output layout")
+        actual = np.asarray(output["output_0__0"])
+        if (actual.shape != reference.shape or actual.dtype not in (np.float16, np.float32)
+                or not np.isfinite(actual).all()):
+            raise ValueError("hosted output violates finite latent contract")
+
+    source_values = reference.astype(np.float64).ravel()
+    device_values = actual.astype(np.float64).ravel()
+    error = source_values - device_values
+    source_norm = float(np.linalg.norm(source_values))
+    device_norm = float(np.linalg.norm(device_values))
+    error_norm = float(np.linalg.norm(error))
+    if source_norm == 0 or device_norm == 0:
+        raise ValueError("zero latent norm cannot define numerical similarity")
+    report = {
+        "scope": "one six-frame Cosmos image encoder on synthetic pattern; no complete InternVLA policy",
+        "status": "component_inference_numeric_measured",
+        "device": inferred["device"],
+        "source_model_id": compiled["source_model_id"],
+        "target_model_id": compiled["target_model_id"],
+        "compile_job_id": compiled["job_id"],
+        "inference_job_id": inferred["job_id"],
+        "input_dataset_id": inferred["input_dataset_id"],
+        "requested_run_options": inferred["options"],
+        "files": files,
+        "output_dtype": str(actual.dtype),
+        "source_vs_device": {
+            "relative_l2": error_norm / source_norm,
+            "snr_db": 20 * math.log10(source_norm / error_norm) if error_norm else None,
+            "max_abs_error": float(np.max(np.abs(error))),
+            "cosine": float(np.dot(source_values, device_values) / (source_norm * device_norm)),
+        },
+        "limits": [
+            "The fixture is synthetic; no real-observation or action-quality tolerance is established.",
+            "The action policy, state lifecycle, admission and complete request were not run on this device.",
+        ],
+    }
+    if args.profile_report is not None:
+        profiled = json.loads(args.profile_report.read_text(encoding="utf-8"))
+        if (profiled.get("status") != "SUCCESS"
+                or profiled.get("model_id") != compiled["target_model_id"]
+                or profiled.get("device") != inferred["device"]
+                or profiled.get("options") != args.expected_run_options):
+            raise ValueError("profile did not execute the same target on the exact device")
+        profile = profiled["profile"]
+        summary = profile["execution_summary"]
+        samples = summary["all_inference_times"]
+        if not samples or any(not isinstance(value, int) or value <= 0 for value in samples):
+            raise ValueError("profile has no valid inference samples")
+        units = Counter(row.get("compute_unit") for row in profile["execution_detail"])
+        if not units:
+            raise ValueError("profile has no compute-unit detail")
+        ordered = sorted(samples)
+        report["profile"] = {
+            "job_id": profiled["job_id"],
+            "sample_count": len(samples),
+            "sample_min_us": ordered[0],
+            "nearest_rank_p50_us": ordered[math.ceil(0.50 * len(samples)) - 1],
+            "nearest_rank_p95_us": ordered[math.ceil(0.95 * len(samples)) - 1],
+            "reported_estimated_inference_time_us": summary["estimated_inference_time"],
+            "reported_peak_memory_bytes": summary["estimated_inference_peak_memory"],
+            "reported_first_load_time_us": summary["first_load_time"],
+            "reported_warm_load_time_us": summary["warm_load_time"],
+            "reported_first_load_peak_memory_bytes": summary["first_load_peak_memory"],
+            "reported_warm_load_peak_memory_bytes": summary["warm_load_peak_memory"],
+            "compute_unit_row_counts": dict(units),
+        }
+        report["status"] = "component_inference_numeric_and_placement_measured"
+    else:
+        report["limits"].append(
+            "Requested NPU is not actual per-node placement without a device profile."
+        )
+    args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"status": report["status"], "difference": report["source_vs_device"]}, indent=2))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/experiments/audit_minicpmo_tts_qualcomm.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""Audit one MiniCPM-o speech-head decode step against pinned ONNX CPU tensors."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+from collections import Counter
+from pathlib import Path
+
+import numpy as np
+
+
+SOURCE_SHA256 = "78cf64804f11ad269ee4180da61588ccc5eefe2942773227345499756e4cc229"
+FIXTURE_SHA256 = "65919ea1f351feec33c1415a3e3db76d60bc9c2beb3d49a72af75dcb418e83b4"
+REFERENCE_SHA256 = "902adcceadbe761d938ef2ade51cf953c8917ea1abfaee33a10c84e300c1bd96"
+COMPILE_OPTIONS = {
+    "--target_runtime qnn_context_binary --quantize_full_type float16",
+    "--target_runtime qnn_dlc --qnn_options default_graph_htp_precision=FLOAT16",
+    "--target_runtime tflite",
+}
+INPUT_NAMES = ["x", "cos", "sin", "mask"] + [
+    name for layer in range(20) for name in (f"k_cache_{layer}", f"v_cache_{layer}")
+]
+OUTPUT_NAMES = ["logits"] + [
+    name for layer in range(20) for name in (f"k_new_{layer}", f"v_new_{layer}")
+]
+
+
+def file_record(path: Path) -> dict:
+    with path.open("rb") as source:
+        digest = hashlib.file_digest(source, "sha256").hexdigest()
+    return {"bytes": path.stat().st_size, "sha256": digest}
+
+
+def read_array(data: np.lib.npyio.NpzFile, name: str, shape: tuple[int, ...]) -> np.ndarray:
+    value = np.asarray(data[name])
+    if value.shape != shape or value.dtype != np.float32 or not np.isfinite(value).all():
+        raise ValueError(f"{name}: expected finite FP32 {shape}, got {value.shape}/{value.dtype}")
+    return value
+
+
+def compare(reference: np.ndarray, actual: np.ndarray) -> dict:
+    source = reference.astype(np.float64).ravel()
+    observed = actual.astype(np.float64).ravel()
+    error = source - observed
+    source_norm = float(np.linalg.norm(source))
+    error_norm = float(np.linalg.norm(error))
+    return {
+        "relative_l2": error_norm / source_norm if source_norm else (0.0 if error_norm == 0 else None),
+        "max_abs_error": float(np.max(np.abs(error))),
+        "source_rms": float(np.sqrt(np.mean(source * source))),
+        "device_rms": float(np.sqrt(np.mean(observed * observed))),
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--device-name", required=True)
+    parser.add_argument("--source-onnx", type=Path, required=True)
+    parser.add_argument("--fixture", type=Path, required=True)
+    parser.add_argument("--cpu-reference", type=Path, required=True)
+    parser.add_argument("--compile-report", type=Path, required=True)
+    parser.add_argument("--inference-report", type=Path, required=True)
+    parser.add_argument("--device-output", type=Path, required=True)
+    parser.add_argument("--output-report", type=Path, required=True)
+    parser.add_argument("--profile-report", type=Path)
+    args = parser.parse_args()
+
+    compiled = json.loads(args.compile_report.read_text(encoding="utf-8"))
+    inferred = json.loads(args.inference_report.read_text(encoding="utf-8"))
+    if compiled.get("status") != "SUCCESS" or inferred.get("status") != "SUCCESS":
+        raise ValueError("compile and inference must be terminal successes")
+    if compiled.get("source_model_id") != "mn4gjjpvn" or compiled.get("options") not in COMPILE_OPTIONS:
+        raise ValueError("compile source/options differ from the pinned speech-head artifact")
+    if inferred.get("model_id") != compiled.get("target_model_id"):
+        raise ValueError("inference used a different target artifact")
+    if compiled["options"] == "--target_runtime tflite":
+        allowed_run_options = {"--compute_unit cpu"}
+    elif compiled["options"].startswith("--target_runtime qnn_dlc"):
+        allowed_run_options = {"--compute_unit npu"}
+    else:
+        # Historical S24 context-binary inference did not record a run option.
+        allowed_run_options = {"", "--compute_unit npu"}
+    if inferred.get("options") not in allowed_run_options:
+        raise ValueError("inference compute request differs from the compiled target route")
+    if inferred.get("input_dataset_id") not in {"d7gwxe3y2", "d7zn8rk57"}:
+        raise ValueError("inference did not use a retained matching fixture dataset")
+    if (compiled.get("device", {}).get("name") != args.device_name
+            or inferred.get("device", {}).get("name") != args.device_name):
+        raise ValueError("jobs did not use the exact requested device")
+
+    files = {
+        name: file_record(path)
+        for name, path in {
+            "source_onnx": args.source_onnx,
+            "fixture": args.fixture,
+            "cpu_reference": args.cpu_reference,
+            "device_output": args.device_output,
+        }.items()
+    }
+    for name, expected in {
+        "source_onnx": SOURCE_SHA256,
+        "fixture": FIXTURE_SHA256,
+        "cpu_reference": REFERENCE_SHA256,
+    }.items():
+        if files[name]["sha256"] != expected:
+            raise ValueError(f"{name} differs from the pinned artifact")
+
+    with np.load(args.fixture, allow_pickle=False) as fixture:
+        if fixture.files != INPUT_NAMES:
+            raise ValueError("fixture input names/order differ")
+        read_array(fixture, "x", (1, 1, 768))
+        for name in ("cos", "sin"):
+            read_array(fixture, name, (1, 1, 1, 64))
+        read_array(fixture, "mask", (1, 1, 1, 257))
+        for name in INPUT_NAMES[4:]:
+            read_array(fixture, name, (1, 12, 256, 64))
+
+    comparisons = {}
+    with np.load(args.cpu_reference, allow_pickle=False) as reference, np.load(
+        args.device_output, allow_pickle=False
+    ) as output:
+        if reference.files != OUTPUT_NAMES:
+            raise ValueError("CPU output names/order differ")
+        expected_keys = [f"output_{index}__0" for index in range(len(OUTPUT_NAMES))]
+        if output.files != expected_keys:
+            raise ValueError("device output names/order differ")
+        for index, name in enumerate(OUTPUT_NAMES):
+            shape = (1, 6562) if index == 0 else (1, 12, 1, 64)
+            source_value = read_array(reference, name, shape)
+            device_value = read_array(output, expected_keys[index], shape)
+            comparisons[name] = compare(source_value, device_value)
+            if name == "logits":
+                comparisons[name]["cpu_top1"] = int(np.argmax(source_value[0]))
+                comparisons[name]["device_top1"] = int(np.argmax(device_value[0]))
+                comparisons[name]["top1_matches"] = bool(
+                    comparisons[name]["cpu_top1"] == comparisons[name]["device_top1"]
+                )
+
+    report = {
+        "scope": "one fixed-cache 20-layer MiniCPM-o speech-head decode step; no complete multimodal stream",
+        "status": "component_inference_numeric_measured",
+        "device": inferred["device"],
+        "compile_job_id": compiled["job_id"],
+        "inference_job_id": inferred["job_id"],
+        "source_model_id": compiled["source_model_id"],
+        "target_model_id": compiled["target_model_id"],
+        "input_dataset_id": inferred["input_dataset_id"],
+        "requested_run_options": inferred["options"],
+        "files": files,
+        "tensor_comparison": comparisons,
+        "cache_max_relative_l2": max(comparisons[name]["relative_l2"] for name in OUTPUT_NAMES[1:]),
+        "limits": [
+            "The historical ONNX export does not attest the exact source checkpoint revision.",
+            "One retained decode fixture cannot establish speech-token or generated-audio quality.",
+            "Thinker, audio/vision encoders, vocoder, state loop and complete Omni request were not tested.",
+        ],
+    }
+    if args.profile_report is not None:
+        profiled = json.loads(args.profile_report.read_text(encoding="utf-8"))
+        expected_profile_options = (
+            "--compute_unit cpu" if compiled["options"] == "--target_runtime tflite"
+            else "--compute_unit npu"
+        )
+        if (profiled.get("status") != "SUCCESS"
+                or profiled.get("model_id") != compiled["target_model_id"]
+                or profiled.get("device") != inferred["device"]
+                or profiled.get("options") != expected_profile_options):
+            raise ValueError("profile did not execute the same target, device and route")
+        summary = profiled["profile"]["execution_summary"]
+        samples = summary["all_inference_times"]
+        if not samples or any(not isinstance(value, int) or value <= 0 for value in samples):
+            raise ValueError("profile has no valid inference samples")
+        units = Counter(row.get("compute_unit") for row in profiled["profile"]["execution_detail"])
+        if not units:
+            raise ValueError("profile has no compute-unit detail")
+        ordered = sorted(samples)
+        report["profile"] = {
+            "job_id": profiled["job_id"],
+            "sample_count": len(samples),
+            "sample_min_us": ordered[0],
+            "nearest_rank_p50_us": ordered[math.ceil(0.50 * len(samples)) - 1],
+            "nearest_rank_p95_us": ordered[math.ceil(0.95 * len(samples)) - 1],
+            "reported_estimated_inference_time_us": summary["estimated_inference_time"],
+            "reported_peak_memory_bytes": summary["estimated_inference_peak_memory"],
+            "reported_first_load_time_us": summary["first_load_time"],
+            "reported_first_load_peak_memory_bytes": summary["first_load_peak_memory"],
+            "compute_unit_row_counts": dict(units),
+        }
+        report["status"] = "component_inference_numeric_and_placement_measured"
+    else:
+        report["limits"].append(
+            "Actual CPU/NPU placement requires a device profile; a request or compiled artifact alone is insufficient."
+        )
+    args.output_report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({
+        "status": report["status"],
+        "device": report["device"]["name"],
+        "logits": comparisons["logits"],
+        "cache_max_relative_l2": report["cache_max_relative_l2"],
+    }, indent=2))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/experiments/audit_qwen38_fp8_copy.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Record file-size and bundled CRC32 checks for a native Qwen3.8 FP8 copy."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import time
+import zlib
+from pathlib import Path
+
+
+def crc32(path: Path) -> str:
+    value = 0
+    with path.open("rb") as stream:
+        while chunk := stream.read(8 * 1024 * 1024):
+            value = zlib.crc32(chunk, value)
+    return f"{value:08x}"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--source", type=Path, required=True)
+    parser.add_argument("--copy", type=Path, required=True)
+    parser.add_argument("--out", type=Path, required=True)
+    args = parser.parse_args()
+    started = time.monotonic()
+    original = {path.name: path.stat().st_size for path in args.source.iterdir() if path.is_file()}
+    copied = {path.name: path.stat().st_size for path in args.copy.iterdir() if path.is_file()}
+    if original != copied:
+        raise ValueError("source and copy differ in file names or sizes")
+    manifest = {}
+    for line in (args.source / "crc32.txt").read_text(encoding="utf-8").splitlines():
+        expected, name = line.split(maxsplit=1)
+        manifest[name] = expected.lower()
+    rows = []
+    for name, expected in manifest.items():
+        actual = crc32(args.copy / name)
+        source_actual = crc32(args.source / name) if actual != expected else None
+        rows.append({"file": name, "bytes": copied[name], "manifest_crc32": expected,
+                     "copy_crc32": actual, "source_crc32_if_manifest_differs": source_actual})
+    weight_rows = [row for row in rows if row["file"].endswith(".safetensors")]
+    mismatches = [row for row in rows if row["manifest_crc32"] != row["copy_crc32"]]
+    if len(weight_rows) != 66 or any(row["manifest_crc32"] != row["copy_crc32"] for row in weight_rows):
+        raise ValueError("at least one safetensors file failed its bundled CRC32")
+    if any(row["source_crc32_if_manifest_differs"] != row["copy_crc32"] for row in mismatches):
+        raise ValueError("copy differs from source on a stale-manifest file")
+    result = {
+        "status": "checked",
+        "source": str(args.source),
+        "copy": str(args.copy),
+        "file_count": len(copied),
+        "total_bytes": sum(copied.values()),
+        "manifest_entries": len(rows),
+        "weight_files_matching_manifest": len(weight_rows),
+        "stale_manifest_nonweight_files": [row["file"] for row in mismatches],
+        "elapsed_s": time.monotonic() - started,
+        "files": rows,
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({key: result[key] for key in (
+        "status", "file_count", "manifest_entries", "weight_files_matching_manifest",
+        "stale_manifest_nonweight_files", "elapsed_s"
+    )}))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/experiments/audit_qwen38_gguf_windows.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Audit pinned Qwen3.8 GGUF Omni CPU/Radeon text and image profiles."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+from pathlib import Path
+
+MODEL_SHA = "c600de0300ae8a0eb3a6c0b8b5561b8b96f16bd2c863c2a66c42de29d391a747"
+MMPROJ_SHA = "2e968a6af97ce35d8971890b257b9b7edabf20ad91450501fa53162a19ee33eb"
+SERVER_SHA = "9ffc5919acb4cb43c7be5f3053b59014cce70a87d3a801fcad49c4f459984f52"
+REVISION = "efbb3b1f70a21d97fd4495240648405f7228554f"
+
+
+def rank(values: list[float], fraction: float) -> float:
+    return sorted(values)[math.ceil(len(values) * fraction) - 1]
+
+
+def read(root: Path, name: str) -> dict:
+    return json.loads((root / name).read_text(encoding="utf-8"))
+
+
+def digest(path: Path) -> dict[str, str | int]:
+    with path.open("rb") as stream:
+        return {"bytes": path.stat().st_size, "sha256": hashlib.file_digest(stream, "sha256").hexdigest()}
+
+
+def check(root: Path, device: str) -> dict:
+    report = read(root, "profile20.json")
+    smoke = read(root, "attempt2.json")
+    placement_verify = read(root, "placement_verify.json")
+    if any(item["status"] != "completed" for item in (report, smoke, placement_verify)):
+        raise ValueError(f"{device}: smoke/profile did not complete")
+    plan = report["execution_plan"]
+    latest_plan = placement_verify["execution_plan"]
+    if (latest_plan["vision_backends"] != (["CPU"] if device == "cpu" else ["Vulkan1"])
+            or latest_plan["model_sha256"] != MODEL_SHA
+            or latest_plan["mmproj_sha256"] != MMPROJ_SHA
+            or any(row["answer"] != expected for row, expected in zip(
+                placement_verify["checks"], ("ready", "Red"), strict=True
+            ))):
+        raise ValueError(f"{device}: current projector placement enforcement failed")
+    if (plan["backend"] != "external.llamacpp.multimodal.v1"
+            or plan["requested_device"] != device
+            or plan["model_sha256"] != MODEL_SHA
+            or plan["mmproj_sha256"] != MMPROJ_SHA
+            or plan["server_sha256"] != SERVER_SHA
+            or plan["request_capacity"] != 1
+            or plan["loaded_rss_bytes"] > report["reserved_host_ram_bytes"]
+            or report["available_host_ram_before_bytes"] < report["reserved_host_ram_bytes"]):
+        raise ValueError(f"{device}: artifact, placement or admission record changed")
+    if device == "cpu":
+        if plan["offloaded_layers"] != ["0", "65"] or plan["model_buffer_devices"] or any(
+            not item.startswith("CPU") for item in plan["all_model_buffers"]
+        ):
+            raise ValueError("CPU model buffers or layer placement changed")
+    elif (plan["expected_device_name"] != "AMD Radeon(TM) 890M Graphics"
+          or plan["offloaded_layers"] != ["65", "65"]
+          or plan["model_buffer_devices"] != ["Vulkan1"]):
+        raise ValueError("Radeon model was not fully offloaded to Vulkan1")
+    ledger = report["ledger_after_shutdown"]
+    if ledger["reserved"].get("host_ram") != 0 or ledger["owners"] or ledger["quarantined"]:
+        raise ValueError(f"{device}: stage reservation was not released")
+    events = []
+    timing = {}
+    for kind, expected in (("text", "ready"), ("image", "Red")):
+        rows = report[f"{kind}_measured"]
+        if len(rows) != 20 or report[f"{kind}_warmup"]["answer"] != expected:
+            raise ValueError(f"{device}: {kind} warmup or sample count changed")
+        if any(row["answer"] != expected or row["kind"] != kind for row in rows):
+            raise ValueError(f"{device}: {kind} output changed")
+        events.extend([report[f"{kind}_warmup"], *rows])
+        walls = [row["wall_s"] for row in rows]
+        p50, p95 = rank(walls, 0.5), rank(walls, 0.95)
+        if (report[f"nearest_rank_p50_{kind}_wall_s"] != p50
+                or report[f"nearest_rank_p95_{kind}_wall_s"] != p95):
+            raise ValueError(f"{device}: {kind} percentile arithmetic changed")
+        timing[kind] = {"n": 20, "p50_s": p50, "p95_s": p95,
+                        "min_s": min(walls), "max_s": max(walls)}
+    events = [*report["checks"], *events]
+    if any(not row["stage_event"]["terminal"]
+           or row["stage_event"]["seq"] != 1
+           or row["stage_event"]["worker_generation"] != plan["worker_generation"]
+           for row in events):
+        raise ValueError(f"{device}: terminal event contract changed")
+    if sorted(row["stage_event"]["epoch"] for row in events) != list(range(1, 45)):
+        raise ValueError(f"{device}: request epochs are missing or duplicated")
+    log = (root / "profile20.server.log").read_text(encoding="utf-8", errors="replace")
+    if device == "cpu":
+        if "offloaded 0/65 layers to GPU" not in log or "CLIP using CPU backend" not in log:
+            raise ValueError("CPU server placement line missing")
+    elif ("using device Vulkan1 (AMD Radeon(TM) 890M Graphics)" not in log
+          or "offloaded 65/65 layers to GPU" not in log
+          or "CLIP using Vulkan1 backend" not in log):
+        raise ValueError("Radeon server placement line missing")
+    return {"startup_s": report["startup_s"], "reserved_host_ram_bytes": report["reserved_host_ram_bytes"],
+            "loaded_rss_bytes": plan["loaded_rss_bytes"], "timing": timing}
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--evidence-root", type=Path, required=True)
+    args = parser.parse_args()
+    root = args.evidence_root
+    cpu = root / "qwen38_gguf_windows_cpu"
+    radeon = root / "qwen38_gguf_windows_radeon"
+    failed = read(cpu, "attempt1.json")
+    if failed["status"] != "failed" or "RSS exceeds stage reservation" not in failed["error"]:
+        raise ValueError("undersized CPU reservation failure is missing")
+    mistyped = read(radeon, "attempt1.json")
+    if mistyped["status"] != "failed" or "differs from the declared artifact hash" not in mistyped["error"]:
+        raise ValueError("mistyped Radeon digest refusal is missing")
+    regression = read(cpu, "spark_text_regression.json")
+    if (regression["status"] != "passed"
+            or regression["execution_plan"]["backend"] != "external.llamacpp.text.v1"
+            or [(row["case"], row["answer"]) for row in regression["checks"]]
+            != [("short", "Paris"), ("inventory_120", "1511")]):
+        raise ValueError("existing Spark llama.cpp text route regressed")
+    summary = {"cpu": check(cpu, "cpu"), "radeon": check(radeon, "Vulkan1")}
+    files = sorted(path for folder in (cpu, radeon) for path in folder.iterdir()
+                   if path.suffix in (".json", ".log") and path.name != "audit_report.json")
+    audit = {
+        "status": "audited_scoped_e2e",
+        "evidence_depth": "P: native Windows Omni complete text and synthetic-image requests on CPU and Radeon 890M",
+        "checkpoint": "ggml-org/Qwen3.8-27B-GGUF",
+        "revision": REVISION,
+        "quantization": "Q4_K_M language model + Q8_0 vision projector",
+        "model_sha256": MODEL_SHA,
+        "mmproj_sha256": MMPROJ_SHA,
+        "server_sha256": SERVER_SHA,
+        "host": "Ryzen AI 9 HX 370, Windows 11 build 26200, Radeon 890M; AC power",
+        "runtime": "Omni checkout through native vLLM 0.29 StageRuntime; llama.cpp b11124 Vulkan build",
+        "scope": "2048 context, 128 max new tokens, concurrency one, one 96x96 red PNG and one short text instruction",
+        "results": summary,
+        "files": {str(path.relative_to(root)): digest(path) for path in files},
+        "limits": [
+            "The prompts produce two completion tokens; no long-context, 128-token output or broad quality claim follows.",
+            "No public AsyncOmni, cancellation/restart, sampled loading/runtime memory peak, power, thermal, concurrency or sustained profile was run for this GGUF route.",
+            "Windows CPU and Radeon use different actual placement, so timings do not establish an isolated accelerator speedup.",
+        ],
+    }
+    (root / "qwen38_gguf_audit_report.json").write_text(json.dumps(audit, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"status": audit["status"], "results": summary}))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/experiments/audit_qwen38_gguf_wsl.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Audit pinned Qwen3.8 GGUF Omni WSL CPU complete-request profiling."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+from pathlib import Path
+
+MODEL_SHA = "c600de0300ae8a0eb3a6c0b8b5561b8b96f16bd2c863c2a66c42de29d391a747"
+MMPROJ_SHA = "2e968a6af97ce35d8971890b257b9b7edabf20ad91450501fa53162a19ee33eb"
+SERVER_SHA = "0d6bb6ade8da8f3231917ce23261da5331333df0007dc4bda74c00e0774e092c"
+REVISION = "efbb3b1f70a21d97fd4495240648405f7228554f"
+
+
+def read(root: Path, name: str) -> dict:
+    return json.loads((root / name).read_text(encoding="utf-8"))
+
+
+def rank(values: list[float], fraction: float) -> float:
+    return sorted(values)[math.ceil(len(values) * fraction) - 1]
+
+
+def digest(path: Path) -> dict[str, str | int]:
+    with path.open("rb") as stream:
+        return {"bytes": path.stat().st_size, "sha256": hashlib.file_digest(stream, "sha256").hexdigest()}
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--evidence-dir", type=Path, required=True)
+    args = parser.parse_args()
+    root = args.evidence_dir
+    for name in ("smoke.json", "smoke2.json"):
+        failed = read(root, name)
+        if failed["status"] != "failed" or "available native host RAM" not in failed["error"]:
+            raise ValueError(f"{name}: low-available-RAM pre-admission refusal missing")
+    repack = read(root, "smoke3.json")
+    if repack["status"] != "failed" or "RSS exceeds stage reservation" not in repack["error"]:
+        raise ValueError("repacked model did not fail the loaded-RSS admission gate")
+    smoke = read(root, "smoke4.json")
+    profile = read(root, "profile20.json")
+    if smoke["status"] != "completed" or profile["status"] != "completed":
+        raise ValueError("WSL no-repack smoke/profile did not complete")
+    plan = profile["execution_plan"]
+    if (plan["backend"] != "external.llamacpp.multimodal.v1"
+            or plan["model_sha256"] != MODEL_SHA
+            or plan["mmproj_sha256"] != MMPROJ_SHA
+            or plan["server_sha256"] != SERVER_SHA
+            or plan["requested_device"] != "cpu"
+            or plan["offloaded_layers"] is not None
+            or not plan["cpu_only_layout"]
+            or plan["model_buffer_devices"]
+            or plan["all_model_buffers"] != ["CPU_Mapped"]
+            or plan["vision_backends"] != ["CPU"]
+            or not plan["disable_repack"]
+            or plan["loaded_rss_bytes"] > profile["reserved_host_ram_bytes"]
+            or profile["available_host_ram_before_bytes"] < profile["reserved_host_ram_bytes"]):
+        raise ValueError("WSL artifact, explicit no-repack CPU placement or admission changed")
+    ledger = profile["ledger_after_shutdown"]
+    if ledger["reserved"].get("host_ram") != 0 or ledger["owners"] or ledger["quarantined"]:
+        raise ValueError("WSL stage did not release its reservation")
+    events = [*profile["checks"]]
+    timing = {}
+    for kind, expected in (("text", "ready"), ("image", "Red")):
+        rows = profile[f"{kind}_measured"]
+        if (len(rows) != 20 or profile[f"{kind}_warmup"]["answer"] != expected
+                or any(row["kind"] != kind or row["answer"] != expected for row in rows)):
+            raise ValueError(f"WSL {kind} sample count/output changed")
+        events.extend([profile[f"{kind}_warmup"], *rows])
+        walls = [row["wall_s"] for row in rows]
+        p50, p95 = rank(walls, 0.5), rank(walls, 0.95)
+        if (profile[f"nearest_rank_p50_{kind}_wall_s"] != p50
+                or profile[f"nearest_rank_p95_{kind}_wall_s"] != p95):
+            raise ValueError(f"WSL {kind} percentile arithmetic changed")
+        timing[kind] = {"n": 20, "p50_s": p50, "p95_s": p95,
+                        "min_s": min(walls), "max_s": max(walls)}
+    if (sorted(row["stage_event"]["epoch"] for row in events) != list(range(1, 45))
+            or any(not row["stage_event"]["terminal"] or row["stage_event"]["seq"] != 1
+                   or row["stage_event"]["worker_generation"] != plan["worker_generation"]
+                   for row in events)):
+        raise ValueError("WSL terminal event epochs missing or duplicated")
+    log = (root / "profile20.server.log").read_text(encoding="utf-8", errors="replace")
+    if ("warning: no usable GPU found" not in log
+            or "CPU_Mapped model buffer size" not in log
+            or "CPU_REPACK model buffer size" in log
+            or "CLIP using CPU backend" not in log):
+        raise ValueError("WSL server log does not confirm CPU/no-repack/vision placement")
+    files = sorted(path for path in root.iterdir() if path.suffix in (".json", ".log")
+                   and path.name != "audit_report.json")
+    audit = {
+        "status": "audited_scoped_e2e",
+        "evidence_depth": "P: WSL CPU Omni complete text and synthetic-image requests",
+        "checkpoint": "ggml-org/Qwen3.8-27B-GGUF",
+        "revision": REVISION,
+        "quantization": "Q4_K_M language model + Q8_0 vision projector",
+        "model_sha256": MODEL_SHA,
+        "mmproj_sha256": MMPROJ_SHA,
+        "server_sha256": SERVER_SHA,
+        "host": "Ryzen AI 9 HX 370, Ubuntu 26.04 on WSL2, 30.91 GiB WSL RAM quota",
+        "runtime": "Omni checkout through Linux vLLM 0.29; llama.cpp b10849 CPU-only build, --no-repack",
+        "scope": "2048 context, 128 max new tokens, concurrency one, one 96x96 red PNG and one short text instruction",
+        "startup_s": profile["startup_s"],
+        "reserved_host_ram_bytes": profile["reserved_host_ram_bytes"],
+        "loaded_rss_bytes": plan["loaded_rss_bytes"],
+        "timing": timing,
+        "files": {path.name: digest(path) for path in files},
+        "limits": [
+            "The prompts produce two completion tokens; long-context, 128-token output and broad quality remain unqualified.",
+            "No public AsyncOmni, cancellation/restart, sampled loading/request memory peak, concurrency or sustained power/thermal profile was run.",
+            "The WSL quota, no-repack option and Linux llama.cpp b10849 differ from the native Windows CPU setup; latencies are not a paired OS comparison.",
+        ],
+    }
+    (root / "audit_report.json").write_text(json.dumps(audit, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"status": audit["status"], "timing": timing}))
+
+
+if __name__ == "__main__":
+    main()