Record InternVLA FP32 AMD NPU compiler failure in matrix

Extract versioned stage contracts, model shared CPU/iGPU/NPU memory constraints, and add explicit graph routes, reservations, output acknowledgements, and worker retirement. Fix native Windows shared-buffer lifetime and host paths. Document qualified hardware paths and remaining rollout gates.

.buildkite/amd/scripts/bootstrap-amd-omni.sh

@@ -14,6 +14,8 @@ set -euo pipefail
 # shellcheck disable=SC1091
 source .buildkite/common/scripts/resolve_skip_ci.sh
 
+AMD_SUITE_SELECTOR=".buildkite/amd/scripts/select_test_suites.py"
+
 if [[ -z "${RUN_ALL:-}" ]]; then
     RUN_ALL=0
 fi
@@ -30,31 +32,20 @@ if [[ -z "${AMD_MIRROR_HW:-}" ]]; then
     AMD_MIRROR_HW="amdproduction"
 fi
 
-fail_fast() {
-    DISABLE_LABEL="ci-no-fail-fast"
-    # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
-    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
-        PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
-        if [[ $PR_LABELS == *"$DISABLE_LABEL"* ]]; then
-            echo false
-        else
-            echo true
-        fi
-    else
-        echo false  # not a PR or BUILDKITE_PULL_REQUEST not set
+PR_LABELS=""
+# Explicit debug suites do not depend on PR labels.
+if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -z "${DEBUG_TEST_YAML:-}" ]]; then
+    if ! PR_LABELS=$(curl -fsSL \
+        "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" \
+        | jq -r '.labels[].name'); then
+        echo "ERROR: Could not read PR labels; refusing to select an incorrect AMD test tier." >&2
+        exit 1
     fi
-}
+fi
 
-check_run_all_label() {
-    RUN_ALL_LABEL="ready-run-all-tests"
-    # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
-    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
-        PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
-        if [[ $PR_LABELS == *"$RUN_ALL_LABEL"* ]]; then
-            echo true
-        else
-            echo false
-        fi
+fail_fast() {
+    if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
+        echo true
     else
         echo false  # not a PR or BUILDKITE_PULL_REQUEST not set
     fi
@@ -64,6 +55,57 @@ if [[ -z "${COV_ENABLED:-}" ]]; then
     COV_ENABLED=0
 fi
 
+resolve_test_specs() {
+    local selected_specs
+    if ! selected_specs=$(python3 "$AMD_SUITE_SELECTOR" \
+        --branch "$BUILDKITE_BRANCH" \
+        --labels "$PR_LABELS" \
+        --debug-test-yaml "${DEBUG_TEST_YAML:-}"); then
+        exit 1
+    fi
+
+    TEST_SPECS=()
+    while IFS= read -r suite_spec; do
+        [[ -z "$suite_spec" ]] && continue
+        TEST_SPECS+=("$suite_spec")
+    done <<< "$selected_specs"
+}
+
+filter_test_specs_by_skip_ci() {
+    local suite_spec level decision
+    local skip_all=0
+    local -a runnable_specs=()
+
+    for suite_spec in "${TEST_SPECS[@]}"; do
+        case "$suite_spec" in
+            READY_TESTS:*) level="l2" ;;
+            MERGE_TESTS:*) level="l3" ;;
+            *)
+                echo "ERROR: unknown AMD test suite spec '$suite_spec'" >&2
+                exit 1
+                ;;
+        esac
+
+        if decision=$(python3 "$SKIP_CI_PY" gate amd "$level"); then
+            echo "Skipping AMD $level suite due to $decision decision."
+            [[ "$decision" == "skip-all" ]] && skip_all=1
+        else
+            runnable_specs+=("$suite_spec")
+        fi
+    done
+
+    TEST_SPECS=("${runnable_specs[@]}")
+    if [[ ${#TEST_SPECS[@]} -eq 0 ]]; then
+        if [[ $skip_all -eq 1 ]]; then
+            buildkite-agent annotate \
+                ":memo: CI skipped — docs or pytest skip-mark changes only" \
+                --style "info" 2>/dev/null || true
+        fi
+        echo "No AMD L2/L3 suites remain after skip-ci filtering."
+        exit 0
+    fi
+}
+
 upload_pipeline() {
     echo "Uploading pipeline..."
     # Install minijinja
@@ -84,7 +126,7 @@ upload_pipeline() {
     # (WIP) Use pipeline generator instead of jinja template
     if [ -e ".buildkite/amd/pipeline_generator/pipeline_generator.py" ]; then
         python -m pip install click pydantic
-        python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
+        python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all="$RUN_ALL" --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
         buildkite-agent pipeline upload .buildkite/amd/pipeline.yaml
         exit 0
     fi
@@ -97,49 +139,12 @@ upload_pipeline() {
 
     cd .buildkite/amd
 
-    # Select test definition file: merge suite for main, ready suite for PRs.
-    # For debugging, DEBUG_TEST_YAML accepts a comma-separated list containing
-    # "merge" and/or "ready" (case-insensitive). Multiple suites are combined
-    # before rendering so they share one amd-build step.
-    if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
-        declare -a DEBUG_TEST_SPECS=()
-        declare -A SEEN_DEBUG_TESTS=()
-        IFS=',' read -ra REQUESTED_DEBUG_TESTS <<< "${DEBUG_TEST_YAML,,}"
-
-        for requested_test in "${REQUESTED_DEBUG_TESTS[@]}"; do
-            # Permit readable values such as "ready, merge," and ignore the
-            # empty item produced by a trailing comma.
-            requested_test="${requested_test//[[:space:]]/}"
-            [[ -z "$requested_test" ]] && continue
-
-            if [[ -n "${SEEN_DEBUG_TESTS[$requested_test]:-}" ]]; then
-                echo "ERROR: duplicate DEBUG_TEST_YAML suite '$requested_test'" >&2
-                exit 1
-            fi
-            SEEN_DEBUG_TESTS[$requested_test]=1
-
-            case "$requested_test" in
-                ready)
-                    DEBUG_TEST_SPECS+=("READY_TESTS:test-amd-ready.yml")
-                    ;;
-                merge)
-                    DEBUG_TEST_SPECS+=("MERGE_TESTS:test-amd-merge.yml")
-                    ;;
-                *)
-                    echo "ERROR: DEBUG_TEST_YAML entries must be 'merge' or 'ready', got '$requested_test'" >&2
-                    exit 1
-                    ;;
-            esac
-        done
-
-        if [[ ${#DEBUG_TEST_SPECS[@]} -eq 0 ]]; then
-            echo "ERROR: DEBUG_TEST_YAML did not contain a test suite" >&2
-            exit 1
-        elif [[ ${#DEBUG_TEST_SPECS[@]} -eq 1 ]]; then
-            TEST_YAML="${DEBUG_TEST_SPECS[0]#*:}"
-        else
-            TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-debug-tests.XXXXXX.yml")
-            python - "$TEST_YAML" "${DEBUG_TEST_SPECS[@]}" <<'PY'
+    # Multiple label-selected or debug-selected suites share one image build.
+    if [[ ${#TEST_SPECS[@]} -eq 1 ]]; then
+        TEST_YAML="${TEST_SPECS[0]#*:}"
+    else
+        TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-selected-tests.XXXXXX.yml")
+        python - "$TEST_YAML" "${TEST_SPECS[@]}" <<'PY'
 import sys
 
 import yaml
@@ -172,13 +177,8 @@ for suite_spec in suite_specs:
 with open(output_path, "w", encoding="utf-8") as output_file:
     yaml.safe_dump(combined, output_file, sort_keys=False)
 PY
-        fi
-        echo "DEBUG_TEST_YAML override: using ${DEBUG_TEST_SPECS[*]}"
-    elif [[ $BUILDKITE_BRANCH == "main" ]]; then
-        TEST_YAML="test-amd-merge.yml"
-    else
-        TEST_YAML="test-amd-ready.yml"
     fi
+    echo "AMD test suites: ${TEST_SPECS[*]}"
 
     (
         set -x
@@ -198,7 +198,7 @@ PY
             > pipeline.yaml
     )
     cat pipeline.yaml
-    if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-debug-tests."*.yml ]]; then
+    if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-selected-tests."*.yml ]]; then
         rm -f -- "$TEST_YAML"
     fi
     buildkite-agent artifact upload pipeline.yaml
@@ -219,12 +219,10 @@ if [[ $BUILDKITE_BRANCH == "main" ]]; then
     file_diff=$(get_diff_main)
 fi
 
-# Early exit: unified skip-ci (docs / skip marks) and CI-yaml-only level targeting.
-if [[ $BUILDKITE_BRANCH == "main" ]]; then
-    gate_bootstrap_ci amd l3
-else
-    gate_bootstrap_ci amd l2
-fi
+# Resolve PR tier labels before skip-ci so ready and merge suites can be
+# filtered independently for CI-YAML-only changes.
+resolve_test_specs
+filter_test_specs_by_skip_ci
 
 patterns=(
     "docker/Dockerfile"
@@ -274,14 +272,6 @@ for file in $file_diff; do
     fi
 done
 
-# Check for ready-run-all-tests label
-LABEL_RUN_ALL=$(check_run_all_label)
-if [[ $LABEL_RUN_ALL == true ]]; then
-    RUN_ALL=1
-    NIGHTLY=1
-    echo "Found 'ready-run-all-tests' label. Running all tests including optional tests."
-fi
-
 # Decide whether to use precompiled wheels
 # Relies on existing patterns array as a basis.
 if [[ -n "${VLLM_USE_PRECOMPILED:-}" ]]; then

.buildkite/amd/scripts/select_test_suites.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Select AMD L2/L3 suites after Buildkite has created an AMD build.
+
+This repository-side selector does not decide which GitHub label events start
+the external ``vllm-omni-amd-ci`` pipeline. Its Buildkite PR build condition
+must admit the relevant label event before this code can inspect the labels.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Iterable
+
+SUITE_SPECS = {
+    "ready": "READY_TESTS:test-amd-ready.yml",
+    "merge": "MERGE_TESTS:test-amd-merge.yml",
+}
+
+
+def _parse_debug_suites(value: str) -> tuple[str, ...]:
+    suites = tuple(item.strip().lower() for item in value.split(",") if item.strip())
+    if not suites:
+        raise ValueError("DEBUG_TEST_YAML did not contain a test suite")
+
+    seen: set[str] = set()
+    for suite in suites:
+        if suite not in SUITE_SPECS:
+            raise ValueError(f"DEBUG_TEST_YAML entries must be 'merge' or 'ready', got {suite!r}")
+        if suite in seen:
+            raise ValueError(f"duplicate DEBUG_TEST_YAML suite {suite!r}")
+        seen.add(suite)
+    return suites
+
+
+def select_amd_test_suites(
+    *,
+    branch: str,
+    labels: Iterable[str],
+    debug_test_yaml: str = "",
+) -> tuple[str, ...]:
+    """Return ordered suite names while preserving legacy unlabeled PRs."""
+
+    if debug_test_yaml:
+        return _parse_debug_suites(debug_test_yaml)
+    if branch == "main":
+        return ("merge",)
+
+    label_set = {label.strip() for label in labels if label.strip()}
+    selected = tuple(suite for label, suite in (("ready", "ready"), ("merge-test", "merge")) if label in label_set)
+    # AMD historically uploaded L2 for every PR build. Keep that behavior for
+    # triggers such as amd-test while adding tier selection for ready/merge-test.
+    return selected or ("ready",)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--branch", required=True)
+    parser.add_argument("--labels", default="")
+    parser.add_argument("--debug-test-yaml", default="")
+    args = parser.parse_args()
+
+    labels = tuple(args.labels.splitlines())
+    try:
+        suites = select_amd_test_suites(
+            branch=args.branch,
+            labels=labels,
+            debug_test_yaml=args.debug_test_yaml,
+        )
+    except ValueError as exc:
+        parser.error(str(exc))
+
+    label_set = {label.strip() for label in labels if label.strip()}
+    if not args.debug_test_yaml and args.branch != "main" and not ({"ready", "merge-test"} & label_set):
+        print(
+            "No AMD L2/L3 tier label found; preserving the legacy ready-suite fallback.",
+            file=sys.stderr,
+        )
+    if "nightly-test" in label_set:
+        print(
+            "AMD nightly-test has no suite yet and does not affect L2/L3 selection.",
+            file=sys.stderr,
+        )
+
+    for suite in suites:
+        print(SUITE_SPECS[suite])
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.buildkite/amd/test-amd-merge.yml

@@ -72,7 +72,22 @@ steps:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
         - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-        - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
+        - >-
+          pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
+          --deselect='tests/diffusion/offloader/test_diffusion_layerwise_offload.py::test_layerwise_offload_diffusion_model[stabilityai/stable-audio-open-1.0]'
+
+    # TODO: Diagnose unstable Stable Audio layerwise-offload memory savings on ROCm.
+    - label: "Diffusion Stable Audio Layerwise Offload Test (quarantined)"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: NonBlocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - >-
+          pytest -s -v
+          'tests/diffusion/offloader/test_diffusion_layerwise_offload.py::test_layerwise_offload_diffusion_model[stabilityai/stable-audio-open-1.0]'
 
     - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs) · Shard 1/2"
       agent_pool: mi300_4
@@ -223,13 +238,10 @@ steps:
       grade: Blocking
       timeout_in_minutes: 50
       commands:
-        - |
-          timeout 40m bash -c '
-            export VLLM_LOGGING_LEVEL=DEBUG
-            export VLLM_WORKER_MULTIPROC_METHOD=spawn
-            export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-          '
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+        - timeout 40m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model and cuda" --run-level "advanced_model"
 
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs

.buildkite/amd/test-amd-ready.yml

@@ -107,6 +107,11 @@ steps:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
         - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
+        - >-
+          timeout 20m pytest -s -v
+          tests/diffusion/models/ltx2/test_ltx2_transformer_ulysses.py
+          -m 'core_model and rocm and cards_2'
+          --run-level "core_model"
 
     - label: "Diffusion GPU Worker Test"
       agent_pool: mi300_2
@@ -122,7 +127,13 @@ steps:
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
-        - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
+        # Preserve legacy one-GPU tests without cards_1 while excluding every
+        # registered multi-GPU marker from this one-GPU job.
+        - >-
+          timeout 15m pytest -sv tests/diffusion/models/
+          -m 'core_model and cuda and not
+          (cards_2 or cards_3 or cards_4 or cards_5 or cards_6 or cards_7 or cards_8)'
+          --run-level "core_model"
 
     - label: "Diffusion · Wan22 Test"
       agent_pool: mi300_1

.buildkite/common/ci_mirror_hardwares.yml

@@ -1,19 +1,36 @@
 # Uploader-only hardware presets for CUDA / NPU test YAML.
 # Referenced as ``mirror_hardwares: l4_1`` on a step; ``upload_pipeline.py`` expands to
-# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins``, then strips
-# ``mirror_hardwares`` before Buildkite upload.
+# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins`` (+ ``retry``
+# for ``l4_*``), then strips ``mirror_hardwares`` before Buildkite upload.
+#
+# Omit the key: compose {chip}_{n} from pytest -m SKU + cards_n (max of several
+# positives; not cards_1 with no positive cards_* → that chip's highest existing preset).
+# Unset MIRROR_HW matches H100 or L4 in -m (both → H100); no match skips the step.
+# MIRROR_HW=b200 matches B200 in -m (else the step is skipped).
+# MIRROR_HW must be empty or b200 (case-insensitive); unknown values fail the upload.
+# CUDA preset strings (h100_4, l4_1) are omitted when MIRROR_HW=b200.
+# Unset MIRROR_HW keeps string presets. NPU presets ignore MIRROR_HW.
 #
 # CUDA naming: {chip}_{count}
-#   l4_1, l4_4, h100_1 … h100_4
+#   l4_1, l4_2, l4_3, l4_4, h100_1 … h100_4, b200_1 … b200_4
 # NPU naming: {queue}_{resource_class}
 #   a2b3_npu_1  — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
 #   a2b3_npu_4  — ascend-a2b3, resource_class npu-4
-#   a3_npu_2    — ascend-a3, resource_class npu-2
+#   a3_npu_2    — ascend-a3, resource_class npu-2 (A3 CI image tag …:$BUILDKITE_COMMIT-a3)
+#   a5_npu_2    — ascend-a5, resource_class npu-2
+#   310p_npu_1  — ascend-310p, resource_class npu-1
 #
-# The ``l4_*`` presets run under the ``docker`` plugin, which does not put
-# ``buildkite-agent`` on PATH inside the container the way the ``kubernetes``
-# presets do, so they set ``mount-buildkite-agent: true``; without it any step
-# calling ``buildkite-agent`` (artifact upload, annotate, meta-data) fails there.
+# B200 (b200-k8s): matches vLLM device ``b200-k8s`` from ci-infra
+# (AgentQueue.B200_K8S + b200_plugin_template). Queue routes the K8s pool;
+# podSpec uses hostNetwork + /raid|/mnt/shared (no H100-style nodeSelector).
+#
+# The ``l4_*`` presets run on the EKS ``l4-k8s`` queue (agent-stack-k8s),
+# packed onto g6.12xlarge L4 nodes (``vllm.ci/gpu-pool=l4x4``). GPU / CPU /
+# memory / shm requests scale with card count (1 GPU = 10 CPU / 40Gi / 24Gi shm).
+# Retry covers lost agents / killed pods, not pytest or OOM (exit 1 / 137).
+# The Kubernetes HF secret deliberately uses VLLM_CI_HF_TOKEN. Scheduled
+# build-level HF_TOKEN values can override pod env entries with the same name;
+# upload_pipeline.py restores the secret-backed value inside the job shell.
 
 x-npu-k8s-plugins: &npu_k8s_plugins
   - kubernetes:
@@ -21,39 +38,224 @@ x-npu-k8s-plugins: &npu_k8s_plugins
         imagePullSecrets:
           - name: swr-secret
 
+# Infra-only retry. Omit exit 1 (pytest failed) and 137 (cgroup OOM / SIGKILL
+# of the test process). ci-infra ``K8S_RETRY`` still retries exit 1.
+x-l4-k8s-retry: &l4_k8s_retry
+  automatic:
+    - exit_status: -1
+      limit: 1
+    - exit_status: 128
+      limit: 1
+    - signal_reason: agent_stop
+      limit: 1
+    - signal_reason: agent_refused
+      limit: 1
+
 
 mirror_hardwares:
   l4_1:
     agents:
-      queue: gpu_1_queue
+      queue: l4-k8s
+    retry: *l4_k8s_retry
     plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          mount-buildkite-agent: true
-          shm-size: "8gb"
-          environment:
-            - HF_HOME=/fsx/hf_cache
-            - HF_TOKEN
-          volumes:
-            - /fsx/hf_cache:/fsx/hf_cache
+      - kubernetes:
+          metadata:
+            annotations:
+              cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+              karpenter.sh/do-not-disrupt: "true"
+          podSpec:
+            priorityClassName: ci
+            serviceAccountName: buildkite-gpu-jobs
+            tolerations:
+              - key: nvidia.com/gpu
+                operator: Exists
+                effect: NoSchedule
+            nodeSelector:
+              vllm.ci/gpu-pool: l4x4
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 1
+                  requests:
+                    nvidia.com/gpu: 1
+                    cpu: "10"
+                    memory: 40Gi
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /fsx/hf_cache
+                env:
+                  - name: HF_HOME
+                    value: /fsx/hf_cache
+                  - name: VLLM_CI_HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+                  sizeLimit: 24Gi
+              - name: hf-cache
+                hostPath:
+                  path: /fsx/hf_cache
+                  type: DirectoryOrCreate
+
+  l4_2:
+    agents:
+      queue: l4-k8s
+    retry: *l4_k8s_retry
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+              karpenter.sh/do-not-disrupt: "true"
+          podSpec:
+            priorityClassName: ci
+            serviceAccountName: buildkite-gpu-jobs
+            tolerations:
+              - key: nvidia.com/gpu
+                operator: Exists
+                effect: NoSchedule
+            nodeSelector:
+              vllm.ci/gpu-pool: l4x4
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 2
+                  requests:
+                    nvidia.com/gpu: 2
+                    cpu: "20"
+                    memory: 80Gi
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /fsx/hf_cache
+                env:
+                  - name: HF_HOME
+                    value: /fsx/hf_cache
+                  - name: VLLM_CI_HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+                  sizeLimit: 48Gi
+              - name: hf-cache
+                hostPath:
+                  path: /fsx/hf_cache
+                  type: DirectoryOrCreate
+
+  l4_3:
+    agents:
+      queue: l4-k8s
+    retry: *l4_k8s_retry
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+              karpenter.sh/do-not-disrupt: "true"
+          podSpec:
+            priorityClassName: ci
+            serviceAccountName: buildkite-gpu-jobs
+            tolerations:
+              - key: nvidia.com/gpu
+                operator: Exists
+                effect: NoSchedule
+            nodeSelector:
+              vllm.ci/gpu-pool: l4x4
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 3
+                  requests:
+                    nvidia.com/gpu: 3
+                    cpu: "30"
+                    memory: 120Gi
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /fsx/hf_cache
+                env:
+                  - name: HF_HOME
+                    value: /fsx/hf_cache
+                  - name: VLLM_CI_HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+                  sizeLimit: 72Gi
+              - name: hf-cache
+                hostPath:
+                  path: /fsx/hf_cache
+                  type: DirectoryOrCreate
 
   l4_4:
     agents:
-      queue: gpu_4_queue
+      queue: l4-k8s
+    retry: *l4_k8s_retry
     plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          mount-buildkite-agent: true
-          shm-size: "8gb"
-          environment:
-            - HF_HOME=/fsx/hf_cache
-            - HF_TOKEN
-          volumes:
-            - /fsx/hf_cache:/fsx/hf_cache
+      - kubernetes:
+          metadata:
+            annotations:
+              cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+              karpenter.sh/do-not-disrupt: "true"
+          podSpec:
+            priorityClassName: ci
+            serviceAccountName: buildkite-gpu-jobs
+            tolerations:
+              - key: nvidia.com/gpu
+                operator: Exists
+                effect: NoSchedule
+            nodeSelector:
+              vllm.ci/gpu-pool: l4x4
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 4
+                  requests:
+                    nvidia.com/gpu: 4
+                    cpu: "40"
+                    memory: 160Gi
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /fsx/hf_cache
+                env:
+                  - name: HF_HOME
+                    value: /fsx/hf_cache
+                  - name: VLLM_CI_HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+                  sizeLimit: 96Gi
+              - name: hf-cache
+                hostPath:
+                  path: /fsx/hf_cache
+                  type: DirectoryOrCreate
 
   h100_1:
     agents:
@@ -74,7 +276,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -109,7 +311,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -144,7 +346,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -179,7 +381,7 @@ mirror_hardwares:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
+                  - name: VLLM_CI_HF_TOKEN
                     valueFrom:
                       secretKeyRef:
                         name: hf-token-secret
@@ -195,6 +397,211 @@ mirror_hardwares:
                   path: /mnt/hf-cache
                   type: DirectoryOrCreate
 
+  # vLLM device: b200-k8s → agents.queue b200-k8s + Buildkite K8s agent-stack podSpec
+  b200_1:
+    agents:
+      queue: b200-k8s
+    plugins:
+      - kubernetes:
+          podSpec:
+            runtimeClassName: nvidia
+            hostNetwork: true
+            dnsPolicy: ClusterFirstWithHostNet
+            imagePullSecrets:
+              - name: k8s-ecr-login-renew-docker-secret
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 1
+                securityContext:
+                  capabilities:
+                    add:
+                      - IPC_LOCK
+                      - SYS_RESOURCE
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: raid
+                    mountPath: /raid
+                  - name: shared
+                    mountPath: /mnt/shared
+                env:
+                  - name: NCCL_CUMEM_HOST_ENABLE
+                    value: "0"
+                  - name: HF_HOME
+                    value: /raid/hf_cache
+                  - name: VLLM_CI_HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: raid
+                hostPath:
+                  path: /raid
+                  type: DirectoryOrCreate
+              - name: shared
+                hostPath:
+                  path: /mnt/shared
+                  type: DirectoryOrCreate
+
+  b200_2:
+    agents:
+      queue: b200-k8s
+    plugins:
+      - kubernetes:
+          podSpec:
+            runtimeClassName: nvidia
+            hostNetwork: true
+            dnsPolicy: ClusterFirstWithHostNet
+            imagePullSecrets:
+              - name: k8s-ecr-login-renew-docker-secret
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 2
+                securityContext:
+                  capabilities:
+                    add:
+                      - IPC_LOCK
+                      - SYS_RESOURCE
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: raid
+                    mountPath: /raid
+                  - name: shared
+                    mountPath: /mnt/shared
+                env:
+                  - name: NCCL_CUMEM_HOST_ENABLE
+                    value: "0"
+                  - name: HF_HOME
+                    value: /raid/hf_cache
+                  - name: VLLM_CI_HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: raid
+                hostPath:
+                  path: /raid
+                  type: DirectoryOrCreate
+              - name: shared
+                hostPath:
+                  path: /mnt/shared
+                  type: DirectoryOrCreate
+
+  b200_3:
+    agents:
+      queue: b200-k8s
+    plugins:
+      - kubernetes:
+          podSpec:
+            runtimeClassName: nvidia
+            hostNetwork: true
+            dnsPolicy: ClusterFirstWithHostNet
+            imagePullSecrets:
+              - name: k8s-ecr-login-renew-docker-secret
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 3
+                securityContext:
+                  capabilities:
+                    add:
+                      - IPC_LOCK
+                      - SYS_RESOURCE
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: raid
+                    mountPath: /raid
+                  - name: shared
+                    mountPath: /mnt/shared
+                env:
+                  - name: NCCL_CUMEM_HOST_ENABLE
+                    value: "0"
+                  - name: HF_HOME
+                    value: /raid/hf_cache
+                  - name: VLLM_CI_HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: raid
+                hostPath:
+                  path: /raid
+                  type: DirectoryOrCreate
+              - name: shared
+                hostPath:
+                  path: /mnt/shared
+                  type: DirectoryOrCreate
+
+  b200_4:
+    agents:
+      queue: b200-k8s
+    plugins:
+      - kubernetes:
+          podSpec:
+            runtimeClassName: nvidia
+            hostNetwork: true
+            dnsPolicy: ClusterFirstWithHostNet
+            imagePullSecrets:
+              - name: k8s-ecr-login-renew-docker-secret
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 4
+                securityContext:
+                  capabilities:
+                    add:
+                      - IPC_LOCK
+                      - SYS_RESOURCE
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: raid
+                    mountPath: /raid
+                  - name: shared
+                    mountPath: /mnt/shared
+                env:
+                  - name: NCCL_CUMEM_HOST_ENABLE
+                    value: "0"
+                  - name: HF_HOME
+                    value: /raid/hf_cache
+                  - name: VLLM_CI_HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: raid
+                hostPath:
+                  path: /raid
+                  type: DirectoryOrCreate
+              - name: shared
+                hostPath:
+                  path: /mnt/shared
+                  type: DirectoryOrCreate
+
   a2b3_npu_1:
     agents:
       queue: "ascend-a2b3"
@@ -244,3 +651,52 @@ mirror_hardwares:
       resource_class: "npu-16"
     image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
     plugins: *npu_k8s_plugins
+
+  a5_npu_2:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-2"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  a5_npu_4:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  a5_npu_8:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-8"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  a5_npu_16:
+    agents:
+      queue: "ascend-a5"
+      resource_class: "npu-16"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a5"
+    plugins: *npu_k8s_plugins
+
+  310p_npu_1:
+    agents:
+      queue: "ascend-310p"
+      resource_class: "npu-1"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-310p"
+    plugins: *npu_k8s_plugins
+
+  310p_npu_2:
+    agents:
+      queue: "ascend-310p"
+      resource_class: "npu-2"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-310p"
+    plugins: *npu_k8s_plugins
+
+  310p_npu_4:
+    agents:
+      queue: "ascend-310p"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-310p"
+    plugins: *npu_k8s_plugins

.buildkite/common/scripts/skip_ci.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Shared Buildkite skip-ci and CI-YAML level targeting logic.
 
 ``resolve_ci_decision(changed_files, ...)`` buckets explicit file lists (tests, dry-run).
@@ -10,12 +11,6 @@ CLI (for bootstrap scripts; exit code is the signal):
   python3 skip_ci.py gate <platform> <level>
       Exit 0 when bootstrap should stop (skip-all, or that L2/L3 target is off).
       Prints ``skip-all`` or ``skip-l23`` on stdout; logs the decision once.
-  python3 skip_ci.py check-skip-all
-      Exit 0 when only docs/skip-mark changes → bootstrap skips entire CI upload.
-  python3 skip_ci.py check-skip-l2-l3 <platform> <level>
-      Exit 0 when that L2/L3 pipeline should be skipped (CI config YAML-only diffs).
-  python3 skip_ci.py print-annotate
-      Print decision message for Buildkite annotation.
 """
 
 from __future__ import annotations
@@ -258,11 +253,10 @@ class CiDecision:
 
 @dataclass(frozen=True)
 class CiContext:
-    """Skip-ci decision plus git diff inputs used to compute it."""
+    """Skip-ci decision plus changed files used for pipeline filtering."""
 
     decision: CiDecision
     changed_files: list[str] | None
-    diff_range: str | None
 
 
 def _log(message: str) -> None:
@@ -332,7 +326,7 @@ def resolve_ci_context_from_git() -> CiContext:
     diff_range = _resolve_diff_range()
     changed_files = _changed_files_for_diff_range(diff_range)
     decision = resolve_ci_decision(changed_files, diff_range=diff_range)
-    return CiContext(decision=decision, changed_files=changed_files, diff_range=diff_range)
+    return CiContext(decision=decision, changed_files=changed_files)
 
 
 # --- Doc / skip-mark helpers ---
@@ -571,23 +565,6 @@ def main() -> int:
     gate_parser.add_argument("platform", help="cuda, amd, intel, or npu")
     gate_parser.add_argument("level", choices=("l2", "l3"), help="test level")
 
-    subparsers.add_parser(
-        "check-skip-all",
-        help="exit 0 when docs/skip-mark-only diff should skip entire CI upload",
-    )
-
-    skip_parser = subparsers.add_parser(
-        "check-skip-l2-l3",
-        help="exit 0 when PLATFORM l2/l3 pipeline should be skipped (CI config YAML-only diffs)",
-    )
-    skip_parser.add_argument("platform", help="cuda, amd, intel, or npu")
-    skip_parser.add_argument("level", choices=("l2", "l3"), help="test level")
-
-    subparsers.add_parser(
-        "print-annotate",
-        help="print human-readable skip reason for Buildkite annotation",
-    )
-
     args = parser.parse_args()
     decision = resolve_ci_context_from_git().decision
 
@@ -600,16 +577,6 @@ def main() -> int:
             return 0
         return 1
 
-    if args.command == "check-skip-all":
-        return 0 if decision.skip_all else 1
-
-    if args.command == "check-skip-l2-l3":
-        return 0 if not decision.is_run(args.platform, args.level) else 1
-
-    if args.command == "print-annotate":
-        print(decision.message or "CI will run normally")
-        return 0
-
     return 1
 
 

.buildkite/common/scripts/upload_pipeline.py

@@ -12,8 +12,19 @@ Bootstrap mode (``bootstrap-upload-steps.yml``):
 
 Test pipeline mode (e.g. test-merge.yml):
   - Drop steps whose ``source_file_dependencies`` do not match changed files.
-  - Expand uploader-only ``mirror_hardwares: l4_1`` into ``agents`` (+ optional ``image``
+  - Expand uploader-only ``mirror_hardwares`` into ``agents`` (+ optional ``image``
     for NPU) + ``plugins`` (see ci_mirror_hardwares.yml).
+  - Omit ``mirror_hardwares`` to compose ``{chip}_{n}`` from pytest ``-m`` SKU
+    markers plus ``cards_n`` (max of several positives; ``not cards_1`` with no
+    positive ``cards_*`` uses that chip's highest existing preset). An explicit
+    preset string (``h100_2``) is used as-is (marks are ignored). Names that
+    are not keys in ``ci_mirror_hardwares.yml`` fail the upload.
+
+    Inferred chip: unset ``MIRROR_HW`` matches ``H100`` or ``L4`` in ``-m``
+    (both → H100); no match skips the step. ``MIRROR_HW=b200`` matches
+    ``B200`` in ``-m`` (otherwise skipped). ``-m`` is not rewritten.
+    ``MIRROR_HW`` must be empty or ``b200``; unknown values fail the upload.
+    A CUDA preset string such as ``h100_4`` is omitted when ``MIRROR_HW=b200``.
 
 Usage:
   python3 upload_pipeline.py [--upload] [--all | --e2e] <pipeline.yml>
@@ -25,11 +36,13 @@ from __future__ import annotations
 
 import argparse
 import copy
+import os
+import re
 import subprocess
 import sys
 from functools import lru_cache
 from pathlib import Path
-from typing import Any
+from typing import Any, Literal
 
 try:
     import yaml
@@ -45,11 +58,27 @@ from skip_ci import (
     resolve_ci_context_from_git,
 )
 
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from tests.helpers.mark import (  # noqa: E402
+    get_skus_for_platform,
+    get_supported_card_counts,
+)
+
 # --- Constants ---
 
 LOG = "upload_pipeline"
 BOOTSTRAP_STEPS_FILENAME = "bootstrap-upload-steps.yml"
-BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset({"image-build", "image-build-a2", "image-build-a3"})
+BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset(
+    {
+        "image-build",
+        "image-build-a2",
+        "image-build-a3",
+        "image-build-a5",
+        "image-build-310p",
+    }
+)
 BOOTSTRAP_UPLOAD_IF_KEYS = {
     "upload-ready-pipeline": "ready",
     "upload-merge-pipeline": "merge",
@@ -58,6 +87,8 @@ BOOTSTRAP_UPLOAD_IF_KEYS = {
 }
 E2E_GROUP_MARKER = "E2E Test"
 CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
+CUDA_HF_TOKEN_ENV = "VLLM_CI_HF_TOKEN"
+CUDA_HF_TOKEN_EXPORT = f'if [ -n "$${{{CUDA_HF_TOKEN_ENV}:-}}" ]; then export HF_TOKEN="$${{{CUDA_HF_TOKEN_ENV}}}"; fi'
 
 # Bootstrap Buildkite ``if`` expressions.
 # ``*_MAIN_IF``: main + env schedule. ``*_LABEL_IF``: PR label (and/or composed with MAIN).
@@ -233,32 +264,268 @@ def _load_mirror_hardwares() -> dict[str, dict[str, Any]]:
     return presets
 
 
-def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any]:
-    """Replace uploader-only ``mirror_hardwares`` with preset fields from ci_mirror_hardwares.yml."""
-    hardware = step.get("mirror_hardwares")
-    if hardware is None:
-        return step
+@lru_cache(maxsize=1)
+def _cuda_mirror_chips() -> tuple[str, ...]:
+    """Lowercase CUDA SKUs that have at least one ``{chip}_{n}`` preset.
 
-    if not isinstance(hardware, str) or not hardware.strip():
+    Longer tokens first so ``h100`` is not matched as a prefix of ``h200``.
+    """
+    skus = get_skus_for_platform("cuda")
+    known = _load_mirror_hardwares()
+    chips = {
+        sku.lower() for sku in skus if any(name == sku.lower() or name.startswith(f"{sku.lower()}_") for name in known)
+    }
+    return tuple(sorted(chips, key=lambda chip: (-len(chip), chip)))
+
+
+def _read_cards_marks(expr: str) -> tuple[set[int], bool]:
+    """Parse registered ``cards_n`` / ``not cards_n`` from a pytest ``-m`` expr.
+
+    Returns positive card counts and whether any ``not cards_*`` is present.
+    """
+    counts = get_supported_card_counts()
+    alt = "|".join(str(n) for n in sorted(counts, key=lambda n: (-len(str(n)), -n))) if counts else "(?!)"
+    positives: set[int] = set()
+    has_not_cards = False
+    for match in re.finditer(rf"\b(?:(not)\s+)?cards_({alt})\b", expr):
+        if match.group(1):
+            has_not_cards = True
+        else:
+            positives.add(int(match.group(2)))
+    return positives, has_not_cards
+
+
+def _read_hardware_marks(expr: str) -> set[str]:
+    """Return lowercase positive CUDA SKUs that have a mirror preset.
+
+    ``not H100`` is ignored. SKUs with no ``{chip}_*`` preset (e.g. H200) are dropped.
+    """
+    skus = sorted(get_skus_for_platform("cuda"), key=lambda s: (-len(s), s))
+    if not skus:
+        return set()
+    alt = "|".join(re.escape(sku) for sku in skus)
+    mirror = set(_cuda_mirror_chips())
+    chips: set[str] = set()
+    for match in re.finditer(rf"\b(?:(not)\s+)?({alt})\b", expr):
+        if match.group(1):
+            continue
+        chip = match.group(2).lower()
+        if chip in mirror:
+            chips.add(chip)
+    return chips
+
+
+# CI policy: not pytest-mark registry.
+_SUPPORTED_MIRROR_HW_SELECTORS = frozenset({"b200"})
+# Unset MIRROR_HW: match H100 then L4 in -m (skip if neither).
+_DEFAULT_INFER_CHIPS = ("h100", "l4")
+_PYTEST_MARKER_ARG = re.compile(r"-m\s+(?:\"([^\"]*)\"|'([^']*)'|(\S+))")
+_CARDS_CHIP_MAX: Literal["max"] = "max"
+
+
+def _get_mirror_hw_selector() -> str:
+    """Return lowercase ``MIRROR_HW``, or empty to keep string presets / marker chips.
+
+    A set value must be ``b200`` (case-insensitive). Unknown values fail closed
+    so a typo cannot silently drop the CUDA pipeline.
+    """
+    selector = os.environ.get("MIRROR_HW", "").strip().lower()
+    if not selector:
+        return ""
+    if selector not in _SUPPORTED_MIRROR_HW_SELECTORS:
+        allowed = " or ".join(repr(s) for s in sorted(_SUPPORTED_MIRROR_HW_SELECTORS))
+        raise ValueError(f"unsupported MIRROR_HW={selector!r}; expected empty or {allowed}")
+    return selector
+
+
+def _read_pytest_marks(commands: Any) -> tuple[set[str], int | Literal["max"] | None]:
+    """Read pytest ``-m`` from step commands: CUDA SKU chips and cards count.
+
+    *chips* are lowercase SKUs from positive markers (``not H100`` is ignored).
+    *cards* is the max positive ``cards_n``, ``"max"`` when only ``not cards_*``
+    is present, or ``None``.
+    """
+    chunks: list[str] = []
+
+    def _collect(value: Any) -> None:
+        if value is None:
+            return
+        if isinstance(value, str):
+            chunks.append(value)
+        elif isinstance(value, list):
+            for part in value:
+                _collect(part)
+        else:
+            chunks.append(str(value))
+
+    _collect(commands)
+    text = "\n".join(chunks)
+
+    chips: set[str] = set()
+    positives: set[int] = set()
+    has_not_cards = False
+    for match in _PYTEST_MARKER_ARG.finditer(text):
+        expr = match.group(1) or match.group(2) or match.group(3) or ""
+        chips |= _read_hardware_marks(expr)
+        card_counts, not_cards = _read_cards_marks(expr)
+        positives.update(card_counts)
+        has_not_cards = has_not_cards or not_cards
+
+    if positives:
+        cards: int | Literal["max"] | None = max(positives)
+    elif has_not_cards:
+        cards = _CARDS_CHIP_MAX
+    else:
+        cards = None
+    return chips, cards
+
+
+def _compose_mirror_hardware_name(
+    chips: set[str],
+    cards: int | Literal["max"] | None,
+    *,
+    step_label: str,
+) -> str | None:
+    """Assemble ``{chip}_{n}`` from pytest ``-m`` chips/cards, or None to skip.
+
+    Unset ``MIRROR_HW``: ``H100`` if in ``-m``, else ``L4``, else skip.
+    Set ``MIRROR_HW``: that chip if in ``-m``, else skip.
+    ``cards`` is a registered ``cards_n``, ``"max"`` for that chip's highest
+    existing preset, or None (error after a chip matched — no ``cards_*``).
+    """
+    selector = _get_mirror_hw_selector()
+    if selector:
+        chip = selector if selector in chips else None
+        skip_reason = f"MIRROR_HW={selector!r} not in -m SKUs {sorted(chips)}"
+    else:
+        chip = next((c for c in _DEFAULT_INFER_CHIPS if c in chips), None)
+        skip_reason = f"no H100 or L4 in -m SKUs {sorted(chips)}"
+    if chip is None:
+        _log(f"skip {step_label}: {skip_reason}")
+        return None
+    if cards is None:
         raise ValueError(
-            f"mirror_hardwares must be a non-empty string in step {_get_step_label(step)!r}",
+            f"step {step_label!r} has a pytest -m SKU marker but no cards_* "
+            f"(or not cards_*); add cards_n or set mirror_hardwares to force a preset",
         )
 
-    preset = _load_mirror_hardwares().get(hardware)
+    count_source = "not cards_*" if cards == _CARDS_CHIP_MAX else f"cards_{cards}"
+    count = None if cards == _CARDS_CHIP_MAX else cards
+    registered = get_supported_card_counts()
+    if count is not None and count not in registered:
+        supported = ", ".join(f"cards_{n}" for n in sorted(registered)) or "(none)"
+        raise ValueError(
+            f"{count_source} in step {step_label!r} is not a registered cards_* mark; supported: {supported}",
+        )
+    known = _load_mirror_hardwares()
+    if count is None:
+        available = [n for n in sorted(registered) if f"{chip}_{n}" in known]
+        if not available:
+            raise ValueError(
+                f"{count_source} in step {step_label!r} has no CUDA count presets for {chip}",
+            )
+        count = max(available)
+    chosen = f"{chip}_{count}"
+    if chosen not in known:
+        raise ValueError(
+            f"{count_source} in step {step_label!r} has no preset {chosen!r}",
+        )
+    _log(f"{step_label}: {count_source} → {chosen!r}")
+    return chosen
+
+
+def _resolve_mirror_hardware_name(hardware: Any, *, step_label: str) -> str | None:
+    """Map ``mirror_hardwares`` to a ``ci_mirror_hardwares.yml`` preset, or None to skip.
+
+    The value is a preset name (``h100_2``). Unknown names fail later when the
+    registry is loaded. CUDA names are omitted when ``MIRROR_HW`` names a
+    different chip; NPU strings ignore ``MIRROR_HW``.
+    """
+    if isinstance(hardware, bool):
+        raise ValueError(f"mirror_hardwares must not be a boolean in step {step_label!r}")
+    if isinstance(hardware, str):
+        name = hardware.strip()
+    elif isinstance(hardware, int):
+        name = str(hardware)
+    else:
+        raise ValueError(
+            f"mirror_hardwares must be a preset string in step {step_label!r}",
+        )
+    if not name:
+        raise ValueError(f"mirror_hardwares must be a non-empty string in step {step_label!r}")
+
+    selector = _get_mirror_hw_selector()
+    token = name.lower()
+    chip = next(
+        (c for c in _cuda_mirror_chips() if token == c or token.startswith(f"{c}_")),
+        None,
+    )
+    if selector and chip is not None and chip != selector:
+        _log(
+            f"skip {step_label}: preset {name!r} is {chip} hardware; MIRROR_HW={selector!r}",
+        )
+        return None
+    return name
+
+
+def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any] | None:
+    """Replace ``mirror_hardwares`` (or infer it from ``-m``) with preset agents/plugins.
+
+    Explicit preset string: used as-is; pytest marks are ignored.
+    Omitted key: match H100/L4 (or ``MIRROR_HW``) in ``-m``, then compose
+    ``{chip}_{n}`` from ``cards_n``. No match skips the step. Steps with no
+    SKU/cards and no ``mirror_hardwares`` are left unchanged (CPU jobs).
+
+    Returns None when the current ``MIRROR_HW`` selector does not apply.
+    """
+    step_label = _get_step_label(step)
+    has_pool = step.get("agents") is not None or step.get("plugins") is not None or step.get("image") is not None
+
+    if "mirror_hardwares" not in step:
+        if has_pool:
+            return step
+        chips, cards = _read_pytest_marks(step.get("commands"))
+        if not chips and cards is None:
+            return step
+        preset_name = _compose_mirror_hardware_name(chips, cards, step_label=step_label)
+    else:
+        if has_pool:
+            raise ValueError(
+                f"step {step_label!r} sets mirror_hardwares together with agents/plugins/image; "
+                f"use mirror_hardwares only",
+            )
+        preset_name = _resolve_mirror_hardware_name(
+            step.get("mirror_hardwares"),
+            step_label=step_label,
+        )
+
+    if preset_name is None:
+        return None
+
+    preset = _load_mirror_hardwares().get(preset_name)
     if preset is None:
         known = ", ".join(sorted(_load_mirror_hardwares()))
         raise ValueError(
-            f"unknown mirror_hardwares {hardware!r} in step {_get_step_label(step)!r}; known: {known}",
-        )
-
-    if step.get("agents") is not None or step.get("plugins") is not None or step.get("image") is not None:
-        raise ValueError(
-            f"step {_get_step_label(step)!r} sets mirror_hardwares together with agents/plugins/image; "
-            "use mirror_hardwares only",
+            f"unknown mirror_hardwares {preset_name!r} in step {step_label!r}; known: {known}",
         )
 
     expanded = copy.deepcopy(preset)
-    return {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
+    merged = {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
+    # A scheduled build's HF_TOKEN may override a pod env entry with the same
+    # name. The preset injects the Kubernetes secret under a collision-proof
+    # alias; restore the conventional name after Buildkite has merged env.
+    if any(preset_name == chip or preset_name.startswith(f"{chip}_") for chip in _cuda_mirror_chips()):
+        commands = merged.get("commands")
+        if isinstance(commands, list):
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, *commands]
+        elif commands is None:
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT]
+        else:
+            merged["commands"] = [CUDA_HF_TOKEN_EXPORT, commands]
+    # Preset retry (K8S_RETRY on l4_*) must not clobber a step that opted out.
+    if "retry" in step:
+        merged["retry"] = step["retry"]
+    return merged
 
 
 def _match_source_file(changed_files: list[str], prefixes: list[str]) -> bool:
@@ -297,7 +564,7 @@ def _process_test_steps(
         nested = step.get("steps")
         if nested is not None:
             kept_nested = _process_test_steps(nested, changed_files)
-            if changed_files is not None and not kept_nested:
+            if not kept_nested:
                 _log(f"omit empty group {_get_step_label(step)!r}")
                 continue
             new_step = {key: value for key, value in step.items() if key != "source_file_dependencies"}
@@ -305,14 +572,11 @@ def _process_test_steps(
             processed.append(new_step)
             continue
 
-        if deps is not None:
-            processed.append(
-                _expand_mirror_hardwares(
-                    {key: value for key, value in step.items() if key != "source_file_dependencies"},
-                ),
-            )
-        else:
-            processed.append(_expand_mirror_hardwares(step))
+        leaf = {key: value for key, value in step.items() if key != "source_file_dependencies"}
+        expanded = _expand_mirror_hardwares(leaf)
+        if expanded is None:
+            continue
+        processed.append(expanded)
 
     return processed
 
@@ -338,6 +602,10 @@ def _render_test_pipeline(
     e2e_only: bool = False,
 ) -> dict[str, Any]:
     """Filter steps by PR diff and strip uploader-only ``source_file_dependencies`` metadata."""
+    # Validate MIRROR_HW once up front. Per-step skip must not run for typos
+    # (e.g. b20o), or a pipeline can shrink to leftover CPU-only steps and
+    # still upload successfully.
+    _get_mirror_hw_selector()
     steps = doc.get("steps")
     if not isinstance(steps, list):
         return doc
@@ -367,10 +635,7 @@ def _render_pipeline(
 
     text = path.read_text(encoding="utf-8")
     ctx = resolve_ci_context_from_git()
-    if force_all or e2e_only:
-        changed_files = None
-    else:
-        changed_files = ctx.changed_files
+    changed_files = None if force_all or e2e_only else ctx.changed_files
 
     doc = yaml.safe_load(text)
     if not isinstance(doc, dict):

.buildkite/cuda/test-merge.yml

@@ -1,6 +1,8 @@
+# CUDA CI L3 (`advanced_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`).
+# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main.
+
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -77,6 +79,9 @@ steps:
     steps:
       - label: "TTS · Qwen3-TTS CustomVoice Test"
         source_file_dependencies:
+          - tests/e2e/features/prefill_phase/
+          - vllm_omni/worker/gpu_model_runner.py
+          - tests/helpers/media.py
           - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
           - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
           - vllm_omni/model_executor/models/qwen3_tts/
@@ -85,17 +90,21 @@ steps:
           - vllm_omni/model_executor/models/common/alias_free_activation.py
           - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
           - vllm_omni/deploy/qwen3_tts.yaml
+          - vllm_omni/entrypoints/openai/serving_speech.py
+          - vllm_omni/entrypoints/openai/tts_adapters/
+          - vllm_omni/engine/stage_pool.py
         commands:
           - |
             timeout 40m bash -c "
               export VLLM_LOGGING_LEVEL=DEBUG
               export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py tests/e2e/features/prefill_phase/test_chunked_prefill_tail.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
         mirror_hardwares: l4_1
 
       - label: "TTS · Qwen3-TTS Base Test"
         source_file_dependencies:
+          - tests/helpers/media.py
           - tests/e2e/online_serving/test_qwen3_tts_base.py
           - tests/e2e/offline_inference/test_qwen3_tts_base.py
           - vllm_omni/model_executor/models/qwen3_tts/
@@ -104,6 +113,9 @@ steps:
           - vllm_omni/model_executor/models/common/alias_free_activation.py
           - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
           - vllm_omni/deploy/qwen3_tts.yaml
+          - vllm_omni/entrypoints/openai/serving_speech.py
+          - vllm_omni/entrypoints/openai/tts_adapters/
+          - vllm_omni/engine/stage_pool.py
           - .buildkite/common/scripts/run_cov_split.sh
           - pyproject.toml
         # 40 -> 60: the coverage split runs the model twice.
@@ -143,6 +155,7 @@ steps:
         source_file_dependencies:
           - tests/e2e/offline_inference/test_minicpmo_4_5.py
           - tests/e2e/online_serving/test_minicpmo_4_5.py
+          - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja
           - vllm_omni/model_executor/models/minicpmo_4_5/
           - vllm_omni/model_executor/models/step_audio2/
           - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
@@ -153,10 +166,11 @@ steps:
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
         # h100_2, not h100_1: advanced_model asserts transcribe every audio
-        # response with Whisper, and tests/helpers/media.py only uses an
-        # accelerator for Whisper when a spare GPU exists. On a single card it
-        # grades on CPU: ~8-10 min per clip vs ~30 s, which alone overruns the
-        # 50-minute job timeout (build 2956).
+        # response with Whisper, and tests/helpers/media.py requires at least
+        # 16 GiB free VRAM on a device to run Whisper on GPU. When the model
+        # server leaves insufficient memory, Whisper falls back to CPU:
+        # ~8-10 min per clip vs ~30 s, which overruns the 50-minute job timeout
+        # (build 2956). h100_2 guarantees a second GPU with ample free VRAM.
         mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Duplex Test"
@@ -172,6 +186,21 @@ steps:
           - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
           - vllm_omni/deploy/minicpmo_4_5.yaml
           - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
+          - vllm_omni/engine/duplex/
+          - vllm_omni/entrypoints/duplex/
+          # Public duplex client library + its unit suites: client-side
+          # changes must re-run the duplex e2e group. The live client probe
+          # (test_duplex_client_live.py, single GPU, core tier) runs in the
+          # ready pipeline's duplex group.
+          - vllm_omni/clients/
+          - tests/clients/
+          - vllm_omni/entrypoints/duplex_request_client.py
+          - vllm_omni/outputs/duplex.py
+          - vllm_omni/model_executor/duplex_sampling.py
+          - tests/engine/duplex/
+          - tests/entrypoints/duplex/
+          - tests/model_executor/models/minicpmo_4_5/duplex/
+          # Legacy Realtime probe client + video stacking used by the e2e drivers.
           - vllm_omni/experimental/fullduplex/
           - vllm_omni/model_executor/models/minicpmo_4_5/
           - vllm_omni/model_executor/models/step_audio2/
@@ -231,3 +260,38 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
             "
         mirror_hardwares: h100_1
+
+      - label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3"
+        source_file_dependencies:
+          - tests/e2e/online_serving/minimax_h3/
+          - tests/helpers/assertions.py
+          - tests/helpers/mark.py
+          - tests/helpers/media.py
+          - tests/helpers/runtime.py
+          - vllm_omni/diffusion/models/minimax_h3/
+          - vllm_omni/diffusion/lora/
+          - vllm_omni/diffusion/offloader/
+          - vllm_omni/diffusion/distributed/
+          - vllm_omni/entrypoints/openai/
+        # Keep four server lifecycles in one Buildkite job, but never in one
+        # pytest process. DLO uses multiprocessing shared-memory queues; the
+        # parent's resource_tracker can retain a stale /psm_* entry during
+        # teardown and strand the next Ref2VA collective.
+        # A per-file process plus a hard timeout also makes a regression fail
+        # instead of leaving the merge queue stuck indefinitely.
+        timeout_in_minutes: 150
+        commands:
+          - |
+            set -eu
+            export VLLM_IMAGE_FETCH_TIMEOUT=60
+            if [ -f vllm_omni/diffusion/models/minimax_h3/lora.py ]; then
+              export VLLM_TEST_MINIMAX_H3_TURBO_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="lightx2v/Minimax-h3-Turbo", filename="minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors"))')
+            fi
+            if [ -f vllm_omni/diffusion/models/minimax_h3/fasth3.py ]; then
+              export VLLM_TEST_MINIMAX_H3_FASTH3_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA", filename="dense-datafree/adapter_model.safetensors"))')
+            fi
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_fl2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_ref2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_turbo_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_fasth3_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+        mirror_hardwares: h100_2

.buildkite/cuda/test-nightly.yml

@@ -1,6 +1,9 @@
+# CUDA CI L4 (`full_model`): nightly and release share this file.
+# Uploaded by Upload Nightly Pipeline: NIGHTLY=1, or PR labels (nightly-test / omni-test / ...).
+# Hardware: omit mirror_hardwares to compose from -m SKU + cards_n; or set a preset string to force.
+
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -13,16 +16,14 @@ steps:
     steps:
 
       - label: ":full_moon: Omni · Function Test with H100 · Single-GPU"
-        timeout_in_minutes: 90
+        timeout_in_minutes: 120
         commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
-        mirror_hardwares: h100_1
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
 
       - label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
         timeout_in_minutes: 90
         commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
-        mirror_hardwares: h100_2
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
 
       - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
         timeout_in_minutes: 50
@@ -30,11 +31,11 @@ steps:
           - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model"
         mirror_hardwares: h100_1
 
-      - label: ":full_moon: Omni · Doc Test with L4"
+      - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU"
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model"
+          - pytest -s -v tests/examples/ -m "full_model and omni and L4 and cards_4" --run-level "full_model"
         mirror_hardwares: l4_4
 
       - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
@@ -178,7 +179,7 @@ steps:
       build.pull_request.labels includes "nightly-test"
     steps:
 
-      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations)"
+      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 2-GPU"
         timeout_in_minutes: 30
         # NOTE: Single-GPU (core_model) tiny model tests run on the ready label,
         # so nightly only adds the multi-GPU parallelism configurations.
@@ -188,7 +189,14 @@ steps:
         # for the CI, since these tests should pass irrespective of whether or not we use
         # the full weights or not.
         commands:
-          - timeout 25m pytest -sv tests/model_tests/diffusion -m 'full_model and cuda' --run-level "core_model"
+          - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_2' --run-level "core_model"
+        mirror_hardwares: l4_2
+
+      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 4-GPU"
+        timeout_in_minutes: 30
+        # CFG+TP extra_test_groups request 2×2=4 devices (see get_required_device_count).
+        commands:
+          - pytest -sv tests/model_tests/diffusion -m 'full_model and cuda and cards_4' --run-level "core_model"
         mirror_hardwares: l4_4
 
   - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
@@ -220,10 +228,21 @@ steps:
             --run-level "full_model"
         mirror_hardwares: h100_2
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Multi-GPU"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Boogu Edit CFG · 3-GPU"
         timeout_in_minutes: 120
         commands:
-          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and not cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
+            -k "test_double_guidance_cfg_parallel and cfg_parallel_3_full"
+            -m "slow and diffusion and cuda and H100 and cards_3"
+            --run-level "full_model"
+        mirror_hardwares: h100_3
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · 4-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and cards_4" --run-level "full_model" --ignore=tests/e2e/accuracy
         mirror_hardwares: l4_4
 
       - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
@@ -259,7 +278,6 @@ steps:
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - export CACHE_DIT_VERSION=1.5.0
           - |
             set +e
@@ -325,6 +343,12 @@ steps:
           - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and cuda" --run-level "full_model"
         mirror_hardwares: h100_2
 
+      - label: ":full_moon: Diffusion X2V · MAGI-2 Preview Serving Smoke"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_magi2.py -m "advanced_model and diffusion and cuda and H100 and cards_4" --run-level "advanced_model"
+        mirror_hardwares: h100_4
+
       - label: ":full_moon: Diffusion X2V · Doc Test"
         timeout_in_minutes: 60
         commands:
@@ -362,7 +386,6 @@ steps:
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - |
             set +e
             pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_1"
@@ -412,7 +435,6 @@ steps:
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - |
             set +e
             pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
@@ -427,10 +449,22 @@ steps:
     depends_on: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
-      - label: ":full_moon: Diffusion · Distributed Test with L4"
+      - label: ":full_moon: Diffusion · Distributed Test with L4 · 2-GPU"
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_2' --run-level "full_model"
+        mirror_hardwares: l4_2
+
+      - label: ":full_moon: Diffusion · Distributed Test with L4 · 3-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_3' --run-level "full_model"
+        mirror_hardwares: l4_3
+
+      - label: ":full_moon: Diffusion · Distributed Test with L4 · 4-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and cards_4' --run-level "full_model"
         mirror_hardwares: l4_4
 
       - label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU"

.buildkite/cuda/test-ready.yml

@@ -1,6 +1,8 @@
+# CUDA CI L2 (`core_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`).
+# Uploaded by Upload Ready Pipeline: PR label `ready`, or main+WEEKLY=1 with --e2e.
+
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -51,17 +53,20 @@ steps:
           - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
         mirror_hardwares: l4_1
 
-      - label: "Diffusion · Distributed & Attention Test"
-        source_file_dependencies:
-          - tests/diffusion/distributed/
-          - tests/diffusion/attention/test_attention_sp.py
-          - vllm_omni/diffusion/distributed/
-          - vllm_omni/diffusion/attention/
-          - vllm_omni/diffusion/data.py
-          - vllm_omni/diffusion/models/
+      - label: "Diffusion · Distributed Test · Single-GPU"
         commands:
-          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model"
-          - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda' --run-level "core_model"
+          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_1' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Distributed Test · 2-GPU"
+        commands:
+          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_2' --run-level "core_model"
+        mirror_hardwares: l4_2
+
+      - label: "Diffusion · Distributed & Attention Test · 4-GPU"
+        commands:
+          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
+          - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda and L4 and cards_4' --run-level "core_model"
         mirror_hardwares: l4_4
 
       - label: "Diffusion · Model Test"
@@ -71,7 +76,7 @@ steps:
 
       - label: "Diffusion · Other Test"
         commands:
-          - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
+          - timeout 15m pytest -sv tests/diffusion/*.py tests/diffusion/ar_diffusion -m 'core_model and cuda' --run-level "core_model"
         mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
@@ -88,7 +93,7 @@ steps:
     commands:
       - |
         timeout 15m bash -c "
-                  pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
+                  pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py -m 'core_model and cuda' --run-level 'core_model'
         "
     mirror_hardwares: l4_1
 
@@ -183,6 +188,7 @@ steps:
       - label: "Omni · MiniCPM-o 4.5 Online Test"
         source_file_dependencies:
           - tests/e2e/online_serving/test_minicpmo_4_5.py
+          - vllm_omni/transformers_utils/chat_templates/minicpmo45_native.jinja
           - vllm_omni/model_executor/models/minicpmo_4_5/
           - vllm_omni/model_executor/models/step_audio2/
           - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
@@ -209,6 +215,21 @@ steps:
           - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
           - vllm_omni/deploy/minicpmo_4_5.yaml
           - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
+          - vllm_omni/engine/duplex/
+          - vllm_omni/entrypoints/duplex/
+          # Public duplex client library + its unit suites: client-side
+          # changes must re-run the duplex e2e group, which includes the
+          # live client probe (test_duplex_client_live.py, single GPU).
+          - vllm_omni/clients/
+          - tests/clients/
+          - tests/e2e/online_serving/test_duplex_client_live.py
+          - vllm_omni/entrypoints/duplex_request_client.py
+          - vllm_omni/outputs/duplex.py
+          - vllm_omni/model_executor/duplex_sampling.py
+          - tests/engine/duplex/
+          - tests/entrypoints/duplex/
+          - tests/model_executor/models/minicpmo_4_5/duplex/
+          # Legacy Realtime probe client + video stacking used by the e2e drivers.
           - vllm_omni/experimental/fullduplex/
           - vllm_omni/model_executor/models/minicpmo_4_5/
           - vllm_omni/model_executor/models/step_audio2/
@@ -226,8 +247,8 @@ steps:
           - vllm_omni/config/stage_config.py
         commands:
           - |
-            timeout 20m bash -c "
-              pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
+            timeout 35m bash -c "
+              pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py tests/e2e/online_serving/test_duplex_client_live.py -m 'core_model and cuda' --run-level 'core_model'
             "
         mirror_hardwares: h100_1
 
@@ -245,6 +266,7 @@ steps:
 
       - label: "TTS · Qwen3-TTS CustomVoice Test"
         source_file_dependencies:
+          - tests/helpers/media.py
           - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
           - vllm_omni/model_executor/models/qwen3_tts/
           - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
@@ -252,6 +274,9 @@ steps:
           - vllm_omni/model_executor/models/common/alias_free_activation.py
           - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
           - vllm_omni/deploy/qwen3_tts.yaml
+          - vllm_omni/entrypoints/openai/serving_speech.py
+          - vllm_omni/entrypoints/openai/tts_adapters/
+          - vllm_omni/engine/stage_pool.py
         commands:
           - |
             timeout 20m bash -c "
@@ -260,25 +285,6 @@ steps:
             "
         mirror_hardwares: l4_1
 
-      - label: "TTS · Qwen3-TTS Base Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen3_tts_base.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
-          - vllm_omni/model_executor/models/common/snake_activation.py
-          - vllm_omni/model_executor/models/common/alias_free_activation.py
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              export VLLM_OMNI_USE_V2_RUNNER=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
-            "
-        mirror_hardwares: l4_1
-
       - label: "Diffusion · Wan22 Test"
         source_file_dependencies:
           - tests/e2e/online_serving/test_wan22_t2v.py

.buildkite/cuda/test-weekly.yml

@@ -1,6 +1,8 @@
+# CUDA CI L5 (`slow` / reliability / extra coverage).
+# Uploaded by Upload Weekly Pipeline: WEEKLY=1 / NON_CRITICAL=1, or PR label `weekly-test`.
+
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -111,17 +113,24 @@ steps:
     if: build.env("NON_CRITICAL") == "1"
     steps:
 
-      # Qwen2.5 expansion (CUDA 4-card / online 2-card) stays on this l4_4 job.
-      - label: "Omni · L4 · Multi-GPU"
+      # Online Qwen2.5 expansion is 2-card; offline expansion + AutoRound are 4-card.
+      - label: "Omni · L4 · 2-GPU"
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and not cards_1" --run-level "core_model"
+          - pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_2" --run-level "core_model"
+        mirror_hardwares: l4_2
+
+      - label: "Omni · L4 · 4-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and cards_4" --run-level "core_model"
           - >-
             pytest -s -v tests/e2e/
             --ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py
             --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-            -m "slow and L4 and omni and not cards_1"
+            -m "slow and L4 and omni and cards_4"
             --run-level "full_model"
         mirror_hardwares: l4_4
 
@@ -194,12 +203,21 @@ steps:
             --run-level "full_model"
         mirror_hardwares: l4_1
 
-      - label: "Diffusion · L4 · Multi-GPU"
+      - label: "Diffusion · L4 · 2-GPU"
         timeout_in_minutes: 60
         commands:
           - >-
             pytest -sv
-            tests/e2e/ -m "slow and diffusion and L4 and not cards_1"
+            tests/e2e/ -m "slow and diffusion and L4 and cards_2"
+            --run-level "full_model"
+        mirror_hardwares: l4_2
+
+      - label: "Diffusion · L4 · 4-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and L4 and cards_4"
             --run-level "full_model"
         mirror_hardwares: l4_4
 

.buildkite/npu/bootstrap-upload-steps.yml

@@ -3,10 +3,10 @@
 steps:
   - label: ":buildkit: Build and Push NPU Test Image - A2"
     key: image-build-a2
-    agents:
+    agents: &npu-image-build-agents
       queue: "ascend-a2b3"
       resource_class: "npu-1"
-    plugins:
+    plugins: &npu-image-build-plugins
       - kubernetes:
           metadata:
             annotations:
@@ -46,123 +46,47 @@ steps:
       IMAGE_NAME: "vllm-omni-ci-npu"
       IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
       BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
-
-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
-
-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}"
 
   - label: ":buildkit: Build and Push NPU Test Image - A3"
     key: image-build-a3
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    plugins:
-      - kubernetes:
-          metadata:
-            annotations:
-              vault.hashicorp.com/agent-init-first: "true"
-              vault.hashicorp.com/agent-inject: "true"
-              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
-              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
-              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
-              vault.hashicorp.com/agent-pre-populate-only: "true"
-              vault.hashicorp.com/agent-run-as-group: "1000"
-              vault.hashicorp.com/agent-run-as-user: "1000"
-              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
-              vault.hashicorp.com/role: ascend-gha-runners
-              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
-              vault.hashicorp.com/tls-skip-verify: "true"
-          podSpecPatch:
-            volumes:
-            - name: token-vol
-              projected:
-                defaultMode: 420
-                sources:
-                - serviceAccountToken:
-                    audience: api
-                    expirationSeconds: 600
-                    path: token
+    agents: *npu-image-build-agents
+    plugins: *npu-image-build-plugins
     env:
       VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3"
       IMAGE_NAME: "vllm-omni-ci-npu"
       IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
       BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}" -a3
 
-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
+  - label: ":buildkit: Build and Push NPU Test Image - A5"
+    key: image-build-a5
+    agents: *npu-image-build-agents
+    plugins: *npu-image-build-plugins
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a5"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}" -a5
 
-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci.a3 \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+  - label: ":buildkit: Build and Push NPU Test Image - 310P"
+    key: image-build-310p
+    agents: *npu-image-build-agents
+    plugins: *npu-image-build-plugins
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-310p"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: bash .buildkite/npu/scripts/build-npu-image.sh Dockerfile.npu.ci "$${VLLM_IMAGE_TAG}" -310p
 
   - label: "Upload Ready Pipeline"
     depends_on:
       - image-build-a2
       - image-build-a3
+      - image-build-a5
+      - image-build-310p
     key: upload-ready-pipeline
     agents:
       queue: "cpu_queue_premerge"
@@ -174,6 +98,8 @@ steps:
     depends_on:
       - image-build-a2
       - image-build-a3
+      - image-build-a5
+      - image-build-310p
     key: upload-nightly-pipeline
     commands:
       - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-nightly.yml

.buildkite/npu/scripts/build-npu-image.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -ex
+
+dockerfile=${1:?Dockerfile is required}
+image_tag=${2:?Image tag is required}
+device_suffix=${3:-}
+
+image_name=${IMAGE_NAME:-vllm-omni-ci-npu}
+image_registry=${IMAGE_REGISTRY:-swr.cn-southwest-2.myhuaweicloud.com/modelfoundry}
+buildkitd_addr=${BUILDKITD_ADDR:-tcp://buildkitd-service.buildkitd:1234}
+
+echo "--- Building and pushing NPU Test Image"
+echo "Image: ${image_registry}/${image_name}:${image_tag}"
+echo "Dockerfile: ${dockerfile}"
+
+if ! command -v buildctl &> /dev/null; then
+  mkdir -p /tmp/buildkit
+  buildkit_version=v0.29.0
+  wget -q \
+    "https://github.com/moby/buildkit/releases/download/${buildkit_version}/buildkit-${buildkit_version}.linux-arm64.tar.gz" \
+    -O /tmp/buildkit.tar.gz
+  tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+  cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+fi
+
+build_args=()
+if [[ -n "${device_suffix}" ]]; then
+  build_args+=(--opt "build-arg:VLLM_ASCEND_DEVICE_SUFFIX=${device_suffix}")
+fi
+
+export DOCKER_CONFIG=/home/user/.docker
+buildctl \
+  --addr="${buildkitd_addr}" \
+  --tlscacert=/home/user/.docker/ca.pem \
+  --tlscert=/home/user/.docker/cert.pem \
+  --tlskey=/home/user/.docker/key.pem \
+  build \
+  --frontend dockerfile.v0 \
+  --local context=. \
+  --local dockerfile=./docker \
+  --opt "filename=${dockerfile}" \
+  "${build_args[@]}" \
+  --secret id=dockerconfig,src=/home/user/.docker/config.json \
+  --output "type=image,name=${image_registry}/${image_name}:${image_tag},push=true" \
+  --progress=plain
+
+echo "--- Image pushed successfully"
+echo "${image_registry}/${image_name}:${image_tag}"

.buildkite/npu/test-npu-ready.yml

@@ -1,26 +1,59 @@
 steps:
-  - label: "Platforms Test"
-    key: ready-platforms-test
+  - group: ":card_index_dividers: Simple Test"
     depends_on: upload-ready-pipeline
-    mirror_hardwares: a3_npu_2
-    commands:
-      - >-
-        pytest -s -v
-        tests/platforms/npu/test_minicpmo_code2wav_npugraph.py
-        -m 'core_model and npu and A3 and omni'
-        --run-level 'core_model'
+    steps:
+      - label: "Platforms Test"
+        key: ready-platforms-test
+        mirror_hardwares: a3_npu_2
+        commands:
+          - >-
+            pytest -s -v
+            tests/platforms/npu/test_minicpmo_code2wav_npugraph.py
+            -m 'core_model and npu and A3 and omni'
+            --run-level 'core_model'
 
-  - label: "Diffusion NPU layer tests"
-    key: ready-diffusion-npu-layer-tests
-    depends_on: upload-ready-pipeline
-    mirror_hardwares: a3_npu_2
-    commands:
-      - >-
-        pytest -s -v
-        tests/diffusion/layers/test_activation.py
-        tests/diffusion/layers/test_fused_qk_norm_rope_npu.py
-        -m 'core_model and npu and A3'
-        --run-level 'core_model'
+      - label: "Diffusion NPU layer tests - A2"
+        key: ready-diffusion-npu-layer-tests-a2
+        mirror_hardwares: a2b3_npu_1
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            -m 'core_model and npu and A2'
+            --run-level 'core_model'
+
+      - label: "Diffusion NPU layer tests - A3"
+        key: ready-diffusion-npu-layer-tests
+        mirror_hardwares: a3_npu_2
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            tests/diffusion/layers/test_fused_qk_norm_rope_npu.py
+            tests/diffusion/models/minimax_h3/test_minimax_h3_qwen3vl_swiglu.py
+            tests/diffusion/models/minimax_h3/test_minimax_h3_modulation.py
+            -m 'core_model and npu and A3'
+            --run-level 'core_model'
+
+      - label: "Diffusion NPU layer tests - A5"
+        key: ready-diffusion-npu-layer-tests-a5
+        mirror_hardwares: a5_npu_2
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            -m 'core_model and npu and A5'
+            --run-level 'core_model'
+
+      - label: "Diffusion NPU layer tests - 310P"
+        key: ready-diffusion-npu-layer-tests-310p
+        mirror_hardwares: 310p_npu_1
+        commands:
+          - >-
+            pytest -s -v
+            tests/diffusion/layers/test_activation.py
+            -m 'core_model and npu and 310P'
+            --run-level 'core_model'
 
   - group: ":card_index_dividers: Omni Model Test"
     key: ready-omni-group

.claude/skills/add-diffusion-model/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: add-diffusion-model
-description: Add a new diffusion model (text-to-image, text-to-video, image-to-video, text-to-audio, image editing) to vLLM-Omni, including Cache-DiT acceleration and parallelism support (TP, SP/USP, CFG-Parallel, HSDP). Use when integrating a new diffusion model, porting a diffusers pipeline or a custom model repo to vllm-omni, creating a new DiT transformer adapter, adding diffusion model support, or enabling multi-GPU parallelism and cache acceleration for an existing model.
+description: Add a new diffusion model (text-to-image, text-to-video, image-to-video, text-to-audio, image editing) to vLLM-Omni, including native non-Diffusers ports, reference-parity validation, Cache-DiT, offload, and parallelism support (TP, SP/USP, CFG-Parallel, HSDP). Use when integrating or reviewing a new diffusion model, porting a Diffusers pipeline or custom model repository, creating a DiT adapter, reusing shared examples, or qualifying multi-GPU and memory optimizations.
 ---
 
 # Adding a Diffusion Model to vLLM-Omni
@@ -26,9 +26,16 @@ Check the model's HF repo for `model_index.json`. This determines your path:
 |----------|----------------|----------------|
 | **Already supported** | `_class_name` in `model_index.json` matches a key in `_DIFFUSION_MODELS` in `registry.py` | Skip implementation, then validate model-specific examples, tests, and docs as needed |
 | **Diffusers-based** | Has standard `model_index.json` with `_diffusers_version`, subfolders for `transformer/`, `vae/`, etc. | Follow **Path A** below |
-| **Custom/private repo** | No diffusers `model_index.json`, weights in non-standard format, custom model code in a separate git repo | Follow **Path B** below |
+| **Native non-Diffusers model** | No Diffusers index, non-standard checkpoint hierarchy, or custom architecture in a separate repo | Follow **Path B** below; port the runtime natively unless an external adapter was explicitly requested |
 | **Hybrid** | Has some diffusers components (VAE) but custom transformer/fusion | Mix of Path A and Path B |
 
+Before coding, write a short integration contract covering runtime ownership,
+checkpoint discovery, reference revision, I/O geometry, attention semantics,
+CFG behavior, auxiliary components, target hardware, and the default deployment.
+For Path B, hybrid models, or any optimization work, read
+[references/native-model-integration-checklist.md](references/native-model-integration-checklist.md)
+and use its phase gates.
+
 ## Path A: Diffusers-Based Model
 
 For models with a standard diffusers layout. See [references/transformer-adaptation.md](references/transformer-adaptation.md) for detailed code patterns.
@@ -103,9 +110,15 @@ For custom loop structures, follow `vllm_omni/diffusion/models/progress_bar.py`
 
 ---
 
-## Path B: Custom/Private Repo Model
+## Path B: Native / Non-Diffusers Model
 
-For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: BAGEL, HunyuanImage3. See [references/custom-model-patterns.md](references/custom-model-patterns.md) for concrete custom-model integration patterns.
+For models without a Diffusers pipeline—weights in custom formats and model
+code in another public or private repository. Treat that repository as a pinned
+correctness oracle. A request for native support means the vLLM-Omni runtime
+must not import the reference implementation; an external adapter is appropriate
+only when the requested scope explicitly permits that dependency. See
+[references/custom-model-patterns.md](references/custom-model-patterns.md) for
+concrete integration patterns.
 
 ### B1. Understand the reference repo
 
@@ -126,16 +139,25 @@ This is the key design decision for custom models. Follow these placement rules:
 | **Pipeline orchestration** (init, forward, denoise loop) | `vllm_omni/diffusion/models/<name>/pipeline_<name>.py` | Always required |
 | **Custom transformer/backbone** (ported and adapted to vllm-omni) | `vllm_omni/diffusion/models/<name>/<name>_transformer.py` or similar | `wan2_2.py`, `fusion.py`, `bagel_transformer.py` |
 | **Custom sub-models** (VAE, fusion, autoencoder) | `vllm_omni/diffusion/models/<name>/` as separate files | `autoencoder.py`, `fusion.py` |
-| **External dependency code** (original repo utilities) | **External repo**, installed via download script or pip | original-repo package via git clone |
+| **Reference-only code** | Keep outside the runtime; use a pinned revision for golden outputs and architecture analysis | Reference inference script |
+| **Explicit external adapter dependency** | External package, only when the requested scope and maintainer direction allow it | Compatibility adapter, not native support |
 | **Hardcoded model configs** | Module-level dicts in pipeline file | `VIDEO_CONFIG`, `AUDIO_CONFIG` dicts |
 | **Download/setup script** | `examples/offline_inference/<name>/download_<name>.py` | `download_<name>.py` |
 | **Custom `model_index.json`** | Generated by download script, placed at model root | Minimal: `{"_class_name": "YourPipeline", ...}` |
 
 ### B3. Handle external dependencies
 
-If the model's code lives in a separate git repo:
+If the model's code lives in a separate git repo, first decide whether the
+requested deliverable is native support or an external adapter. Do not silently
+choose the adapter path.
 
-**Option 1: Import with graceful fallback** (recommended for models with external utils)
+**Option 1: Port the code directly** (default for native support)
+
+Copy the essential model files into `vllm_omni/diffusion/models/<name>/` and
+adapt them to shared vLLM-Omni contracts. Keep checkpoint loading strict and
+use the pinned reference only to generate parity evidence.
+
+**Option 2: Import with graceful fallback** (adapter scope only)
 
 ```python
 try:
@@ -147,11 +169,9 @@ except ImportError:
     )
 ```
 
-**Option 2: Port the code directly** (preferred when feasible)
-
-Copy the essential model files into `vllm_omni/diffusion/models/<name>/` and adapt them. This avoids external dependencies. BAGEL does this — `autoencoder.py` and `bagel_transformer.py` are ported directly.
-
-**Decision criteria**: Port if the code is self-contained and won't diverge. Use external deps if the model repo is actively maintained and the code is complex.
+Use an external runtime dependency only when the user explicitly requested an
+adapter or a maintainer approved the exception. Document the dependency,
+pinned revision, installation path, and unsupported native features.
 
 ### B4. Handle custom weight loading
 
@@ -221,7 +241,8 @@ class CustomPipeline(nn.Module):
 
 ### B5. Create the `model_index.json`
 
-Custom models need a `model_index.json` at the model root for vllm-omni to discover them. For custom models, this is minimal:
+Prefer a `model_index.json` at the model root when vLLM-Omni owns an assembled
+checkpoint directory. For custom models, this is minimal:
 
 ```json
 {
@@ -230,7 +251,14 @@ Custom models need a `model_index.json` at the model root for vllm-omni to disco
 }
 ```
 
-The `_class_name` must match a key in `_DIFFUSION_MODELS` in `registry.py`. Additional keys are model-specific (accessed via `od_config.model_config`).
+The `_class_name` must match a key in `_DIFFUSION_MODELS` in `registry.py`.
+Additional keys are model-specific (accessed via `od_config.model_config`).
+
+If the released repository is immutable and has neither a root `config.json`
+nor a Diffusers index, add it to a generic native-checkpoint signature resolver:
+match the exact Hub ID or a distinctive set of local files and return the
+pipeline class. Do not use model-name substrings or add parallel one-off
+predicates in CLI, config, and serving consumers.
 
 If the model's weights come from multiple HF repos, write a **download script** that:
 1. Downloads from each repo
@@ -292,7 +320,9 @@ Use the appropriate existing example script:
 | Image-to-Image | `examples/offline_inference/image_to_image/image_edit.py` |
 | Text-to-Audio | `examples/offline_inference/text_to_audio/text_to_audio.py` |
 
-For custom/Omni models that don't fit these categories, create a dedicated example script.
+Reuse these shared scripts even for custom models when their request and output
+contracts fit. Create a dedicated model script only when the shared category
+cannot represent the protocol, and document that gap in the PR.
 
 **Validation**: No errors, output is meaningful, quality matches reference implementation.
 
@@ -300,13 +330,17 @@ See [references/troubleshooting.md](references/troubleshooting.md) for common er
 
 ### Step 6: Add Example Scripts
 
-For Omni or custom models, create:
+Only when the shared category scripts cannot represent the model, create:
 - `examples/offline_inference/your_model_name/` — offline script + README
 - `examples/online_serving/your_model_name/` — server script + client
 - Download script if weights require assembly from multiple sources
 
 ### Step 7: Update Documentation
 
+Follow the [`add-recipe` skill](../add-recipe/SKILL.md) to add or update the
+model-family recipe and its `recipes/README.md` row with verified specifications,
+hardware, commands, feature links, and qualification evidence.
+
 Required updates:
 1. `docs/user_guide/diffusion/parallelism/overview.md` — parallelism support overview/table
 2. `docs/user_guide/diffusion/cpu_offload.md` — if CPU offload supported (add to supported models table)
@@ -346,107 +380,20 @@ Classify the model's **CI priority** first:
 
 **Keep model-specific code inside test modules — not `tests/helpers/{slug}.py`:** deploy constants, prompts, sampling dicts, and inline `request_config` / `form_data` belong in each `test_{slug}.py` and `test_{slug}_expansion.py`. Do not add per-model files under `tests/helpers/`; reuse only repo-wide harness (`mark`, `media`, `runtime`, `stage_config`, `assertions`). **L2+ online/offline e2e:** reuse or add `send_*_request` in `tests/helpers/runtime.py` — tests call the handler, not raw `omni.generate` / HTTP. See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers**.
 
+Keep the model suite proportional using the six distinct failure owners in the
+native integration checklist. Combine supported features into a few
+parametrized E2E rows instead of creating one test per optimization.
+
 ### Step 9: Add Cache-DiT Acceleration
 
-Cache-DiT accelerates inference by caching intermediate computation results across denoising steps. After your model is working correctly on a single GPU, add cache-dit support.
+Add caching only after uncached single-device correctness. Read
+[references/cache-dit-patterns.md](references/cache-dit-patterns.md), use the
+automatic single-block-list path when possible, and add a registered
+`BlockAdapter` only for genuinely custom block topology.
 
-See [references/cache-dit-patterns.md](references/cache-dit-patterns.md) for detailed code patterns.
-
-#### 9a. Determine your model type
-
-| Model Type | Description | Action |
-|------------|-------------|--------|
-| **Standard single-transformer** | One transformer with one `ModuleList` of blocks | No code needed — `CacheDiTBackend` auto-detects via `enable_cache_for_dit()` |
-| **Multi-block-list** | One transformer with multiple block lists (e.g., `transformer_blocks` + `single_transformer_blocks`) | Write custom enabler with `BlockAdapter` |
-| **Dual-transformer** | Two transformers (e.g., high-noise + low-noise) | Write custom enabler with `BlockAdapter` wrapping both |
-
-#### 9b. Standard models — verify automatic support
-
-For standard single-transformer models, test directly:
-
-```python
-omni = Omni(
-    model="your-model-name",
-    cache_backend="cache_dit",
-    cache_config={
-        "Fn_compute_blocks": 1,
-        "Bn_compute_blocks": 0,
-        "max_warmup_steps": 4,
-    }
-)
-```
-
-Check logs for "Cache-dit enabled successfully on xxx". If it works, skip to Step 9e.
-
-#### 9c. Custom architectures — write a custom enabler
-
-For multi-block-list or dual-transformer models, write a custom enabler function:
-
-```python
-from cache_dit import BlockAdapter, ForwardPattern, ParamsModifier, DBCacheConfig
-
-def enable_cache_for_your_model(pipeline, cache_config):
-    db_cache_config = DBCacheConfig(
-        num_inference_steps=None,
-        Fn_compute_blocks=cache_config.Fn_compute_blocks,
-        Bn_compute_blocks=cache_config.Bn_compute_blocks,
-        max_warmup_steps=cache_config.max_warmup_steps,
-        max_cached_steps=cache_config.max_cached_steps,
-        max_continuous_cached_steps=cache_config.max_continuous_cached_steps,
-        residual_diff_threshold=cache_config.residual_diff_threshold,
-    )
-
-    cache_dit.enable_cache(
-        BlockAdapter(
-            transformer=pipeline.transformer,
-            blocks=[
-                pipeline.transformer.transformer_blocks,
-                pipeline.transformer.single_transformer_blocks,
-            ],
-            forward_pattern=[ForwardPattern.Pattern_1, ForwardPattern.Pattern_1],
-            params_modifiers=[ParamsModifier(...)],
-        ),
-        cache_config=db_cache_config,
-    )
-
-    def refresh_cache_context(pipeline, num_inference_steps, verbose=True):
-        cache_dit.refresh_context(
-            pipeline.transformer, num_inference_steps=num_inference_steps, verbose=verbose
-        )
-    return refresh_cache_context
-```
-
-#### 9d. Register the custom enabler
-
-Add your enabler to `CUSTOM_DIT_ENABLERS` in
-`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
-
-```python
-CUSTOM_DIT_ENABLERS = {
-    "Wan22Pipeline": enable_cache_for_wan22,
-    "LongCatImagePipeline": enable_cache_for_longcat_image,
-    "YourModelPipeline": enable_cache_for_your_model,  # Add here
-}
-```
-
-#### 9e. Test Cache-DiT
-
-```python
-omni = Omni(
-    model="your-model-name",
-    cache_backend="cache_dit",
-    cache_config={
-        "Fn_compute_blocks": 1, "Bn_compute_blocks": 0,
-        "max_warmup_steps": 4, "residual_diff_threshold": 0.24,
-    }
-)
-images = omni.generate("a beautiful landscape",
-    OmniDiffusionSamplingParams(num_inference_steps=50))
-```
-
-**Verify**: 1) logs show cache enabled, 2) 1.5-2x speedup, 3) output quality acceptable vs baseline.
-
-If quality degrades, lower `residual_diff_threshold` (try 0.12-0.18) or increase `max_warmup_steps` (try 6-8).
+Verify a real cache hit and compare quality with the uncached baseline. Make a
+speed claim only at a realistic step count where warmup permits hits; an
+all-warmup smoke proves integration, not acceleration.
 
 ---
 
@@ -460,107 +407,30 @@ See [references/parallelism-patterns.md](references/parallelism-patterns.md) for
 
 #### 10a. Tensor Parallelism (TP)
 
-Shards DiT linear layers across GPUs. Requires code changes in the transformer.
-
-**What to change in the transformer**:
-1. Replace `nn.Linear` with `ColumnParallelLinear` / `RowParallelLinear` / `QKVParallelLinear`
-2. Update `load_weights()` to handle QKV fusion with `stacked_params_mapping`
-3. Use `self.to_qkv.num_heads` (local heads) instead of total heads for split sizes
-
-```python
-from vllm.model_executor.layers.linear import (
-    QKVParallelLinear, RowParallelLinear, ColumnParallelLinear,
-)
-
-# Attention: QKV → RowParallel output
-self.to_qkv = QKVParallelLinear(dim, head_dim, num_heads, num_kv_heads)
-self.to_out = RowParallelLinear(dim, dim, input_is_parallel=True)
-
-# FFN: ColumnParallel → RowParallel
-self.w1 = ColumnParallelLinear(dim, ffn_dim)
-self.w2 = RowParallelLinear(ffn_dim, dim, input_is_parallel=True)
-```
-
-**Constraints**: `num_heads % tp_size == 0` and `num_kv_heads % tp_size == 0`.
-
-**Test**: `--tensor-parallel-size 2`
+Replace compatible projections with vLLM parallel linears, preserve checkpoint
+fusion/loading, and use local head counts. Require query/KV head divisibility
+and compare a multi-rank forward with the one-rank oracle.
 
 #### 10b. Sequence Parallelism (SP / USP)
 
-Splits sequence tokens across GPUs. Non-intrusive via `_sp_plan` on the transformer class — no changes to `forward()`.
-
-**What to change in the transformer**:
-
-Add `_sp_plan` class attribute:
-
-```python
-from vllm_omni.diffusion.distributed.sp_plan import (
-    SequenceParallelInput, SequenceParallelOutput,
-)
-
-class YourTransformer(nn.Module):
-    _sp_plan = {
-        "blocks.0": {
-            "hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3),
-        },
-        "proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
-    }
-```
-
-If inline tensor ops (e.g., `torch.cat`) exist between shard/gather points, extract them into `nn.Module` submodules so hooks can intercept them.
-
-For RoPE that needs splitting, add an entry for the RoPE module with `split_output=True`.
-
-**Test**: `--ulysses-degree 2` (offline) or `--usp 2` (online serving)
+Prefer the declarative `_sp_plan`. For packed variable-length attention or
+learned-sink LSE correction, keep model math explicit and reuse shared exchange
+utilities. Validate uneven sequence splits, RoPE coordinates, and outputs
+against the one-rank oracle.
 
 #### 10c. CFG Parallel
 
-Distributes positive/negative CFG branches across 2 GPUs. Requires the pipeline to inherit `CFGParallelMixin`.
-
-**What to change in the pipeline**:
-
-```python
-from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
-
-class YourPipeline(nn.Module, CFGParallelMixin):
-    def diffuse(self, ...) -> torch.Tensor:
-        for i, t in enumerate(timesteps):
-            positive_kwargs = {...}
-            negative_kwargs = {...} if do_true_cfg else None
-            noise_pred = self.predict_noise_maybe_with_cfg(
-                do_true_cfg=do_true_cfg, true_cfg_scale=cfg_scale,
-                positive_kwargs=positive_kwargs, negative_kwargs=negative_kwargs,
-            )
-            latents = self.scheduler_step_maybe_with_cfg(
-                noise_pred, t, latents, do_true_cfg
-            )
-        return latents
-```
-
-Override `predict_noise()` if your transformer call is non-standard. Override `combine_cfg_noise()` for multi-output models (e.g., video + audio).
-
-**Constraint**: Exactly 2 GPUs. Only for models using classifier-free guidance.
-
-**Test**: `--cfg-parallel-size 2`
+Confirm the model uses CFG. Then reuse `CFGParallelMixin`, overriding prediction
+or recombination only for non-standard or multi-output pipelines. Distinguish a
+packed positive/negative implementation from two independent branches and
+validate the two-rank result against the packed one-rank oracle.
 
 #### 10d. HSDP (Hybrid Sharded Data Parallel)
 
-Shards transformer weights via PyTorch FSDP2 to reduce per-GPU VRAM. No code changes to the forward pass — just add a class attribute.
-
-**What to change in the transformer**:
-
-```python
-class YourTransformer(nn.Module):
-    @staticmethod
-    def _is_transformer_block(name: str, module) -> bool:
-        return "blocks" in name and name.split(".")[-1].isdigit()
-
-    _hsdp_shard_conditions = [_is_transformer_block]
-```
-
-**Constraint**: Cannot combine with TP. For standalone HSDP, set `hsdp_shard_size` explicitly.
-
-**Test**: `--use-hsdp` or `DiffusionParallelConfig(use_hsdp=True)`
+Declare layer shard conditions and ignored rank-local modules; preserve mixed
+checkpoint dtypes. HSDP cannot combine with TP. Measure parameter loading,
+FSDP materialization, warm HBM, and host PSS rather than assuming sharding saves
+peak memory. Keep the resident layout as default if HSDP is worse.
 
 #### 10e. Update parallelism documentation
 
@@ -599,6 +469,12 @@ Pipelines without `SupportsComponentDiscovery` fall back to scanning
 well-known attribute names (`transformer`, `text_encoder`, `vae`,
 etc.), which fails for non-standard names.
 
+Keep model-specific checkpoint paths, nested block aliases, layout transforms,
+and component lifecycles in the model package. Change a shared offloader only
+for a general contract, demonstrate another consumer or a framework-level bug,
+and add one focused shared regression. Avoid `if ModelName` branches in shared
+backends.
+
 ### Step 12: Performance Profiling
 
 After verifying correctness and implementing parallelism/caching, profile the model's performance to identify bottlenecks and ensure optimal execution.
@@ -608,6 +484,12 @@ See the [Profiling Single-Stage Diffusion](../../../docs/contributing/profiling.
 2. Using Nsight Systems (`nsys`) with `profiler: "cuda"` for low-overhead CUDA traces.
 3. Controlling profiling via `omni.start_profile()` and `omni.stop_profile()`.
 
+Report cold E2E, warm user latency, steady-state wave time, peak allocated and
+reserved HBM, host PSS for the full process tree, and stage boundaries. State
+whether prompt encoding and VAE/audio decoding are included. For multi-device
+layouts, plot user latency against throughput per device and keep different
+denoising-step counts on separate Pareto frontiers.
+
 ---
 
 ## Pre-commit conventions
@@ -635,7 +517,7 @@ shellcheck). In particular:
 3. **Use small models**: Test with smaller variants first
 4. **Check tensor shapes**: Most errors are reshape mismatches in attention
 5. **Add features incrementally**: Single GPU → TP → SP → CFG → HSDP → Cache-DiT
-6. **For custom models**: Get the model running with the original code first, then progressively replace components with vllm-omni equivalents
+6. **For custom models**: Run the pinned reference separately, then port the runtime natively; do not ship temporary imports from the reference implementation
 7. **Cache-DiT before parallelism tuning**: Cache-DiT is lossy — verify quality at baseline before combining with parallelism
 8. **Combine lossless + lossy**: e.g., TP + SP + Cache-DiT for maximum throughput
 
@@ -644,6 +526,7 @@ shellcheck). In particular:
 - [vllm-omni-test skill](../vllm-omni-test/SKILL.md) — L1–L4 markers, naming, Buildkite wiring, run commands
 - [Transformer Adaptation](references/transformer-adaptation.md) — porting transformers from diffusers
 - [Custom Model Patterns](references/custom-model-patterns.md) — patterns for non-diffusers models
+- [Native Model Integration Checklist](references/native-model-integration-checklist.md) — ownership boundaries, phase gates, qualification matrix, and review evidence
 - [Parallelism Patterns](references/parallelism-patterns.md) — TP, SP/USP, CFG parallel, HSDP implementation details
 - [Cache-DiT Patterns](references/cache-dit-patterns.md) — cache-dit acceleration for standard and custom architectures
 - [Troubleshooting](references/troubleshooting.md) — common errors and fixes

.claude/skills/add-diffusion-model/references/cache-dit-patterns.md

@@ -9,7 +9,9 @@ Three caching strategies:
 - **TaylorSeer**: Calibration-based prediction using Taylor expansion to estimate block outputs
 - **SCM** (Step Computation Masking): Dynamic step skipping based on configurable policies
 
-**Typical speedup**: 1.5-2.5x depending on model and configuration.
+Speedup is model-, step-, and policy-dependent. Treat a cache-enabled run as
+integration evidence until logs or a focused test show real hits and repeated
+timings show an improvement over the uncached baseline.
 
 **Official docs**: https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/cache_dit
 
@@ -243,8 +245,10 @@ vllm serve your-model --omni --port 8098 \
 
 **Verification checklist**:
 1. Logs show "Cache-dit enabled successfully on xxx"
-2. Performance: 1.5-2x speedup vs no cache
-3. Quality: compare output with `cache_backend=None`
+2. A focused test or summary confirms at least one real cache hit
+3. Quality is compared with `cache_backend=None`
+4. Performance uses a realistic step count, repeated timings, and a policy
+   whose warmup does not consume the whole run
 
 ## Excluded Models
 

.claude/skills/add-diffusion-model/references/custom-model-patterns.md

@@ -16,7 +16,7 @@ vllm_omni/diffusion/models/wan2_2/
 
 The transformer is loaded separately via `weights_sources` + `load_weights()`. Non-transformer components (VAE, text encoder) are loaded in `__init__` via `from_pretrained()`.
 
-### Custom model with external deps
+### External adapter (not native support)
 
 ```
 vllm_omni/diffusion/models/<name>/
@@ -28,7 +28,10 @@ examples/offline_inference/<task>/
 └── download_<name>.py             # Downloads weights and/or clones code repo
 ```
 
-All weights loaded eagerly in `__init__`. `load_weights()` is a no-op. External dependency imported with try/except.
+All weights are loaded eagerly in `__init__`. `load_weights()` is a no-op. The
+external dependency is imported with try/except. Use this pattern only when the
+requested scope explicitly allows an adapter; it does not satisfy a request for
+a native vLLM-Omni implementation.
 
 ### Custom model with ported code (e.g., BAGEL)
 
@@ -86,7 +89,7 @@ init → load ALL weights eagerly via custom helpers → load_weights() = no-op
 - `load_weights()` is `pass`
 - Weights may come from multiple HF repos in different formats (`.pth`, `.safetensors`)
 
-Use this when:
+Use this only for an explicitly approved external adapter when:
 - The original model has complex, well-tested loading code you don't want to rewrite
 - Weights span multiple HF repos
 - Weight format is non-standard (e.g., a single `.pth` file, not sharded safetensors)
@@ -112,7 +115,14 @@ Custom model `model_index.json` (minimal):
 }
 ```
 
-The only **required** field is `_class_name` — it must match a key in `_DIFFUSION_MODELS` in `registry.py`. Other fields are model-specific and accessible via `od_config.model_config` dict.
+The only **required** field is `_class_name` — it must match a key in
+`_DIFFUSION_MODELS` in `registry.py`. Other fields are model-specific and
+accessible via `od_config.model_config`.
+
+When the released repository cannot be modified and contains neither this file
+nor a root `config.json`, use the generic native-checkpoint signature resolver.
+Match the exact Hub ID or a distinctive local file set; never infer support from
+a model-name substring.
 
 ## External Dependency Management
 
@@ -141,6 +151,10 @@ def download_dependency():
 
 Copy essential files from the original repo into `vllm_omni/diffusion/models/<name>/`. Adapt imports to use vllm-omni utilities. Benefits: no external dependency, no git clone step. Drawback: must maintain the ported code.
 
+This is the default for native model support. Keep the reference repository at
+a pinned revision for architecture analysis and golden outputs, but do not
+import it from production code.
+
 ## Multi-Modal Input/Output Protocols
 
 Custom models that handle images, audio, or video I/O should implement protocol classes:

.claude/skills/add-diffusion-model/references/native-model-integration-checklist.md

@@ -0,0 +1,131 @@
+# Native Diffusion Model Integration Checklist
+
+Use this checklist for native non-Diffusers models, hybrid pipelines, and
+model-specific optimization work.
+
+## 1. Freeze the contract before coding
+
+Record:
+
+- Reference repository and immutable revision.
+- Official checkpoint ID, local layout, license, and required auxiliary files.
+- Pipeline stages: prompt/image encoding, denoising, video/image/audio decoding,
+  and muxing.
+- Token packing, RoPE coordinates, attention sinks/LSE behavior, MoE layout,
+  scheduler equations, CFG formulation, dtypes, and checkpoint transforms.
+- Output geometry, frame count/FPS, audio rate/channels, supported tasks, and
+  default generation steps.
+- Available local hardware versus the official qualification hardware.
+- Intended default deployment and optional optimization paths.
+
+State whether the deliverable is a native implementation or an external
+adapter. For native support, use the reference as an oracle only.
+
+## 2. Use phase gates
+
+| Phase | Required exit evidence |
+|---|---|
+| Architecture | Component/weight inventory; unsupported reference dependencies identified; ownership decisions documented |
+| Native correctness | Strict checkpoint load; tiny forward or one-step golden vs pinned reference; output contract validated |
+| Framework integration | Registry/discovery, shared request types, existing category example, serving route, docs, and recipe work without model-name branching in shared scripts |
+| Lossless distribution | One-rank oracle vs each supported TP/SP/CFG/HSDP/VAE layout; released-checkpoint smoke for the recommended layout |
+| Memory optimization | Ordinary/layerwise/DLO behavior, peak HBM, full-tree host PSS, and stage timeline; shared backend unchanged unless a general contract is proven |
+| Lossy optimization | Cache hit demonstrated; quality delta measured; speed claim made only when warmup permits hits |
+| Full qualification | Default-resolution E2E at official step count for every supported task; media metadata and hashes recorded |
+| Review readiness | Focused tests, Ruff/format, strict docs, DCO, current-main rebase, issue/RFC for large changes, concise PR evidence and limitations |
+
+Do not start performance tuning before native one-step parity. Do not claim a
+feature from topology validation alone when released weights can exercise it.
+
+## 3. Keep ownership narrow
+
+Default placement:
+
+| Concern | Owner |
+|---|---|
+| Architecture, checkpoint remap, packed attention math, nested block alias, auxiliary lifecycle | Model package |
+| Kernel/version dispatch already useful to multiple models | Shared attention utility |
+| Request-dependent example defaults | Typed model-extra entry consumed through a registry |
+| Exact-ID/local-file detection for repositories without standard metadata | Generic native-checkpoint signature table |
+| General offload/cache/HSDP bug | Shared layer plus one focused shared regression |
+
+Before changing shared code, answer:
+
+1. Is this a framework contract or only this checkpoint's layout?
+2. Can the model expose an existing declarative hook instead?
+3. Is there a second consumer or a framework-level failing test?
+4. Can the shared change avoid importing or naming the model?
+
+If the answers do not justify a shared change, keep it model-local.
+
+## 4. Reuse examples without leaking model logic
+
+- Reuse the existing `x_to_y.py` category script.
+- Resolve the pipeline class through standard metadata or native signature
+  discovery, not path substrings.
+- Put dynamic model defaults and validation in `model_extras`; let shared
+  examples consume only the generic typed API.
+- Do not create image/speech/default abstractions without a real consumer.
+- Add a dedicated example only when the shared request/output protocol cannot
+  express the model.
+
+## 5. Build a bounded evidence matrix
+
+Start with the recommended resident layout. Add only supported, useful points:
+
+- Single request: TP, SP, or hybrid TP/SP.
+- Throughput: DLO/DP layouts when concurrent requests are valid.
+- CFG: compare CFG-parallel output with the packed-CFG oracle only if the model
+  actually uses CFG.
+- HSDP: record loading and materialization peaks; it can use more HBM than a
+  resident SP layout.
+- VAE patch parallelism: compare decoded tensors or media exactly where the
+  algorithm should be lossless.
+- Cache: separate cache integration from acceleration evidence.
+
+With four devices, qualify four-device layouts such as DP4, DP2SP2, SP4, or
+CFG2xSP2. Do not present CFG2xSP4 or an official eight-device runtime as locally
+validated.
+
+For each measured layout record:
+
+- Cold E2E and warm user latency.
+- Steady-state wave time and throughput per device.
+- Prompt, denoise, video decode, audio decode, and mux times.
+- Peak allocated/reserved HBM and full-process-tree host PSS.
+- Step count, seed, output geometry, frame/FPS/audio metadata, and parity metric.
+
+Plot latency versus throughput/device. Keep one-step, four-step, and full-step
+results separate because they are not quality-equivalent.
+
+## 6. Keep tests proportional
+
+Prefer a small set with distinct failure ownership:
+
+1. Strict weight-key/dtype/load contract.
+2. Tiny native golden or one-step reference parity.
+3. One parametrized distributed-parity test covering supported layouts.
+4. Pipeline/request/output geometry and serving smoke.
+5. Shared-example/default test when the category script changes.
+6. One shared regression only for a genuine shared-layer fix.
+
+Avoid duplicating the same shape or configuration assertion across
+model-specific files. Keep expensive released-checkpoint evidence in the PR or
+recipe unless the repository's CI tier explicitly requires it.
+
+## 7. Write evidence, not marketing
+
+In the PR description include:
+
+- A compact architecture/data-flow explanation.
+- Default deployment and why it is recommended.
+- Code-change counts by component.
+- Accuracy and distributed-parity evidence tied to commands/scripts.
+- Separate Pareto figures for different step counts.
+- Supported versus validated versus planned optimizations.
+- Hardware gaps and untested official-runtime claims.
+
+Call a result a bounded local qualification unless it includes repeated timing
+and variance. State whether latency includes text/image encoding and all
+decoders. Preserve the reference revision and exact commands needed to
+reproduce the evidence.

.claude/skills/add-recipe/SKILL.md

@@ -0,0 +1,126 @@
+---
+name: add-recipe
+description: Add or update an in-repository vLLM-Omni model recipe with verified task, input, output, hardware, command, feature, and validation contracts. Use when creating files under recipes/, restructuring a recipe after review, documenting a newly supported model, or synchronizing recipe claims with support tables and shared feature guides.
+---
+
+# Add a vLLM-Omni Recipe
+
+## Source the contract
+
+Read [`recipes/TEMPLATE.md`](../../../recipes/TEMPLATE.md) and
+[`recipes/README.md`](../../../recipes/README.md) before editing. Inspect the
+canonical model card/repository, implementation, shared examples, tests, and
+available local qualification evidence.
+
+Do not infer support from a model name, sibling recipe, registry entry, or
+topology validator. Mark unexecuted configurations as configuration-only.
+
+## Route by model family
+
+Read only the contributor and user documentation relevant to the model:
+
+| Family | Contributor guide | User-facing documentation to synchronize |
+|---|---|---|
+| Diffusion | [`adding a diffusion model`](../add-diffusion-model/SKILL.md) | `docs/user_guide/diffusion_features.md`, applicable diffusion feature guides, and shared image/video/audio examples |
+| TTS | [`adding a TTS model`](../add-tts-model/SKILL.md) | `examples/offline_inference/text_to_speech/README.md`, `examples/online_serving/text_to_speech/README.md`, and `docs/serving/speech_api.md` |
+| Omni | [`adding an omni model`](../../../docs/contributing/model/adding_omni_model.md) | model-family offline/online example docs, `docs/serving/chat_completions_api.md`, and `docs/user_guide/feature_compatibility.md` |
+
+All families must update `docs/models/supported_models.md` and the matching row
+in `recipes/README.md`. Do not add a modality-specific support document when
+the repository has no such table; update the closest shared user contract.
+
+## Structure the recipe
+
+Keep the recipe task-oriented and use one hardware-specific file named
+`recipes/<vendor>/<model>-<hardware>.md`. A recipe may contain multiple
+deployment topologies on that hardware, but each additional accelerator model
+or platform requires its own suffixed file.
+
+1. **Summary:** vendor, exact model ID, runtime, modes, named hardware,
+   recommended deployment, and maintainer.
+2. **Supported model contract:** put task, input, output, and provided-profile
+   tables before setup or commands.
+3. **References:** link the canonical model source, shared examples, supported
+   model table, and feature matrix.
+4. **Checkpoint/setup:** pin revisions when possible and state required assets.
+5. **Hardware:** document the single accelerator named by the file suffix,
+   including every validated device-count/topology profile on it.
+6. **Commands:** reuse shared `examples/` entrypoints unless the model contract
+   truly requires a dedicated script. Include offline and online paths that
+   were validated.
+7. **Supported features:** use a compact model-specific topology/status table
+   with links to shared feature guides.
+8. **Verification:** provide a quick command and exact expected output
+   contract.
+9. **Qualification evidence:** report correctness, memory, and timing evidence
+   with its measurement scope and caveats.
+
+## Separate hardware from software
+
+For every locally validated hardware profile, record:
+
+- accelerator vendor/model and per-device memory;
+- number of devices;
+- interconnect (`NVLink`, `PCIe`, or the platform equivalent);
+- host memory when CPU staging/offload is material;
+- whether the profile is runtime-qualified or configuration-only.
+
+Record OS, Python, driver/runtime, framework versions, and the vLLM-Omni
+revision in a separate software-environment table. Keep precision, worker
+count, and DP/TP/SP/PP sizes with the deployment profile or exact command.
+
+Never generalize a result from one accelerator family to another or combine
+NVIDIA, AMD, NPU, or distinct accelerator models in one recipe. Create a
+separate hardware-suffixed recipe and `recipes/README.md` row. Distinguish
+upstream requirements from hardware exercised by the PR.
+
+## Keep shared documentation concise
+
+Link the applicable diffusion, TTS, omni, serving, or design guide for generic
+feature semantics and launch instructions. The recipe should contain only the
+model-specific status, valid topology, required flag difference, and evidence
+boundary.
+
+Put unsupported combinations in the same feature table. Avoid repeating the
+same model-specific prose below the global feature matrix; the recipe is the
+detailed source of truth.
+
+## Report evidence precisely
+
+For every performance or memory value, state:
+
+- checkpoint/revision and workload;
+- accelerator and device count;
+- precision and topology;
+- cold/warm scope, step count, and concurrency;
+- peak HBM and host-memory metric when measured;
+- output/quality guard and sample count;
+- monitoring overhead or other known variance.
+
+Call a single bounded run qualification evidence, not a benchmark. Do not add
+a benchmark script unless the contribution explicitly requires one.
+
+## Synchronize repository documentation
+
+Update all applicable locations:
+
+- the row in `recipes/README.md`;
+- `docs/models/supported_models.md`;
+- the family-specific documentation selected in **Route by model family**;
+- shared example documentation when commands or flags change.
+
+Keep global support tables compact. Link to the recipe instead of duplicating
+its deployment explanation.
+
+## Validate
+
+Run, at minimum:
+
+```bash
+pre-commit run --files <changed recipe/docs/skill files>
+mkdocs build --strict
+git diff --check
+```
+
+Remove generated documentation artifacts after validation. Confirm every
+recipe link resolves and every claimed profile has matching code or evidence.

.claude/skills/add-tts-model/SKILL.md

@@ -499,6 +499,13 @@ Common pre-commit failures, recovery commands for missing sign-off, and the
 full `pre-commit run` invocation for a TTS model:
 [references/precommit-dco.md](references/precommit-dco.md).
 
+## Documentation and recipe
+
+Follow the [`add-recipe` skill](../add-recipe/SKILL.md) to add or update the
+in-repository model-family recipe, the `recipes/README.md` index, the supported
+models table, TTS example documentation, and the speech API contract using
+validated evidence.
+
 ## Integration Checklist
 
 Use this checklist when integrating a new TTS model:
@@ -543,6 +550,7 @@ Use this checklist when integrating a new TTS model:
 - [ ] Buildkite entries match level: `test-ready.yml` / `test-merge.yml` or nightly TTS job / `test-nightly.yml`
 - [ ] Gradio demo working
 - [ ] Documentation added (offline + online docs, nav, supported models)
+- [ ] In-repository recipe and `recipes/README.md` row added via the `add-recipe` skill
 
 ### Phase 4: Async Chunk
 - [ ] Pipeline declares the async handoff processor

.claude/skills/production-add-diffusion-model/SKILL.md

@@ -0,0 +1,458 @@
+---
+name: production-add-diffusion-model
+description: Productionize a vLLM-Omni diffusion model after its Day-0 vertical slice works. Use when work requires official API parity and input limits, feature-combination evidence, online FP8, distributed layerwise offload, per-request Cache-DiT, CUDA/ROCm/NPU/XPU recipes, faster or sparse attention, fused operators, USP or disaggregation analysis, continuous batching and abort handling, long-running RPS stability, or accuracy/performance/reliability CI. For initial architecture porting, registry wiring, and basic weight loading, use add-diffusion-model first.
+---
+
+# Productionizing a Diffusion Model
+
+## Scope
+
+Use this skill to move a working model integration from Day-0 support to a
+measured, fail-closed production deployment. It is deliberately separate from
+[`add-diffusion-model`](../add-diffusion-model/SKILL.md); do not modify or copy
+that skill when productionizing a model. If registry, pipeline, or basic loader
+work is incomplete, read that skill first and finish the Day-0 vertical slice.
+
+MiniMax-H3 [PR #5691](https://github.com/vllm-project/vllm-omni/pull/5691)
+is the merged Day-0 case study. Its open optimization
+[issue #5700](https://github.com/vllm-project/vllm-omni/issues/5700) is an
+actively maintained roadmap and evidence map, not a list of already supported
+features. It can lag merged implementation: read its current revision, cited
+PRs, target source, and maintained recipes, then leave every unverified row
+`not tested` until the target revision has scoped evidence.
+
+Read these references before the corresponding gate:
+
+- [API and hardware recipes](references/api-and-recipes.md)
+- [Production feature patterns](references/feature-patterns.md)
+- [Performance patterns and code examples](references/performance-patterns.md)
+- [Serving, soak, and CI validation](references/production-validation.md)
+
+## Non-negotiable evidence contract
+
+Create a matrix whose row key is at least:
+
+```text
+task x API mode x execution mode/capacity x shape/schedule x attention backend x
+packed layout x cache policy x quantization x offload mode x topology x
+hardware x dtype x output representation/transport
+```
+
+Use exactly these states:
+
+| State | Meaning |
+|---|---|
+| `validated` | The exact row passed correctness and measured deployment tests, with reproducible artifacts. |
+| `limited` | A bounded subset passed; the limitation and rejection/fallback behavior are explicit. |
+| `unsupported` | A stable architectural/platform restriction is proven and fails early with an actionable error. |
+| `not tested` | No adequate evidence exists. This is the default. |
+
+Never infer compatibility from import success, server startup, another task,
+another card, or another topology. Record the model/checkpoint revision,
+vLLM-Omni commit, commands, request assets and hashes, seed/schedule/shape,
+raw output artifacts, environment, and result for every `validated` row.
+
+## Workflow
+
+Follow the gates in order. Keep a dense BF16 single-device path as the
+correctness oracle until every advertised fast path has parity evidence.
+
+### Gate 0: Freeze the official contract
+
+Pin the official implementation and checkpoint revision. Build a matrix for:
+
+- constructor/call arguments, defaults, scheduler and sigma schedule;
+- task-to-source modality/count/order rules;
+- prompt, MIME, bytes, pixels, frames, duration, FPS, steps, seed, guidance,
+  output count, and output format limits;
+- offline API and every public sync/async serving endpoint;
+- checkpoint partitions and task selection.
+
+Normalize offline, sync, and async requests through one validation contract.
+Reject an invalid task/source combination before downloads, upload persistence,
+temporary files, async job creation, or engine submission. Use 400 for invalid
+semantics and 413 for payload limits; clean partial resources on every failure.
+See [API and hardware recipes](references/api-and-recipes.md).
+
+### Gate 1: Re-prove correctness under production shapes
+
+1. Make loading strict: track expected, loaded, unexpected, and intentionally
+   ignored tensors. A fused destination is complete only after every source
+   shard arrives. Missing retained parameters abort startup.
+2. Fix reference revision, asset hashes, seed, schedule, and shape. Compare
+   component outputs, intermediate denoise latents, and final artifacts.
+3. Test unequal packed samples. Verify `cu_seqlens`, padding exclusion,
+   modality/CFG ownership, split/gather boundaries, and one output per request.
+4. Test partial process groups with non-member ranks and group size smaller
+   than world size. Do not construct a private group per component or request.
+5. Run negative API cases before positive E2E tests to prove side-effect-free
+   rejection and cleanup.
+
+### Gate 2: Wire shared fast primitives
+
+Prefer explicit, testable vLLM-Omni/vLLM operators with native fallbacks.
+`torch.compile` remains a separately validated regional optimization; it is
+not a substitute for correct operator selection or lifecycle-safe boundaries.
+
+#### RMSNorm + RoPE baseline
+
+Use the shared cross-platform operators, preserving the official Q/K norm and
+RoPE order:
+
+```python
+from vllm_omni.diffusion.layers.norm import RMSNorm
+from vllm_omni.diffusion.layers.rope import (
+    RotaryEmbedding,
+    apply_rope_to_qk,
+)
+
+self.norm_q = RMSNorm(head_dim, eps=eps)
+self.norm_k = RMSNorm(head_dim, eps=eps)
+self.rope = RotaryEmbedding(is_neox_style=False)
+
+query = self.norm_q(query)  # [B, S, Hq, D] or packed [T, Hq, D]
+key = self.norm_k(key)      # [B, S, Hkv, D] or packed [T, Hkv, D]
+query, key = apply_rope_to_qk(
+    self.rope,
+    query,
+    key,
+    (cos, sin),
+)
+```
+
+Verify `is_neox_style`, `half_head_dim`, partial rotary dimensions, cos/sin
+layout, and packed row ownership against the official model. `False` means
+interleaved/GPT-J style; do not assume the default is NeoX. Shared RMSNorm and
+RoPE are two dispatched operators, not automatically one fused kernel.
+
+For packed non-interleaved RoPE, use the shared fused boundary when its contract
+matches the model:
+
+```python
+from vllm_omni.diffusion.layers.fused_qk_norm_rope import fused_qk_norm_rope
+
+query, key = fused_qk_norm_rope(
+    query,                    # [T, Hq, D]
+    key,                      # [T, Hkv, D]
+    self.norm_q.weight,       # [D]
+    self.norm_k.weight,       # [D]
+    rope_table,               # [T, rotary_dim] = [cos | sin]
+    self.norm_q.variance_epsilon,
+)
+```
+
+The current CUDA fast path specializes BF16 `head_dim=128`, `rotary_dim=96`;
+the public function keeps an eager fallback for unsupported inputs. Verify the
+packed row/frequency contract and trace the actual fast path. Do not reshape an
+incompatible official RoPE layout merely to enter this kernel.
+
+MiniMax-H3-style partial NeoX RoPE rotates a prefix and passes the suffix
+through:
+
+```python
+self.q_norm = RMSNorm(head_dim, eps=eps, dtype=torch.bfloat16)
+self.k_norm = RMSNorm(head_dim, eps=eps, dtype=torch.bfloat16)
+self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)
+
+def apply_partial_rope(self, x, freqs, rot_dim):
+    x_rot, x_pass = x[..., :rot_dim], x[..., rot_dim:]
+    cos, sin = torch.cos(freqs).to(x.dtype), torch.sin(freqs).to(x.dtype)
+    return torch.cat((self.rope(x_rot, cos, sin), x_pass), dim=-1)
+```
+
+The partial-RoPE snippet assumes `import torch` and lives on the owning
+attention module; adapt names and validated frequency shapes rather than
+copying it at module scope.
+
+Call the modules normally so `CustomOp` selects CUDA/HIP/NPU/XPU/native
+implementations. Do not call `forward_cuda()` directly. Compare the shared
+path with native BF16 at operator, block, denoise-trajectory, and artifact
+levels before claiming either accuracy or speed.
+
+#### Faster BF16 attention and fused projections
+
+Use local TP head counts and role-aware shared attention:
+
+```python
+from vllm.model_executor.layers.linear import QKVParallelLinear
+from vllm_omni.diffusion.attention.layer import Attention
+
+self.qkv_proj = QKVParallelLinear(
+    hidden_size=hidden_size,
+    head_size=head_dim,
+    total_num_heads=num_heads,
+    total_num_kv_heads=num_kv_heads,
+    bias=False,
+    quant_config=quant_config,
+    prefix=f"{prefix}.qkv_proj",
+)
+self.attn = Attention(
+    num_heads=self.qkv_proj.num_heads,
+    num_kv_heads=self.qkv_proj.num_kv_heads,
+    head_size=head_dim,
+    softmax_scale=head_dim**-0.5,
+    causal=False,
+    qkv_layout="BSND",
+    role="self",
+    prefix=prefix,
+)
+```
+
+Prove the selected backend from runtime logs/trace and compare it with dense
+SDPA. Packed metadata must express real sample boundaries; do not build masks
+that the selected backend ignores. A backend that is faster for one shape or
+platform is not a global default. See [performance patterns](references/performance-patterns.md)
+for QKV, SwiGLU, AdaLN, sparse attention, redundancy, and fusion examples.
+
+### Gate 3: Add online FP8 without breaking the loader
+
+Route the active config to each component, then pass a stable runtime prefix
+and config into every quantizable vLLM linear:
+
+```python
+def resolve_component_quant_config(quant_config, component):
+    return quant_config.resolve(component) if hasattr(quant_config, "resolve") else quant_config
+
+transformer_quant_config = resolve_component_quant_config(
+    od_config.quantization_config,
+    "transformer",
+)
+self.transformer = MyDiT(
+    od_config,
+    quant_config=transformer_quant_config,
+    prefix="transformer",
+)
+
+self.qkv_proj = QKVParallelLinear(
+    hidden_size=hidden_size,
+    head_size=head_dim,
+    total_num_heads=num_heads,
+    total_num_kv_heads=num_kv_heads,
+    quant_config=quant_config,
+    prefix=f"{prefix}.qkv_proj",
+)
+```
+
+Use `build_quant_config()` for direct construction/testing:
+
+```python
+from vllm_omni.quantization import build_quant_config
+
+quant_config = build_quant_config({
+    "transformer": {"method": "fp8"},
+    "text_encoder": None,
+    "vae": None,
+    "default": None,
+})
+```
+
+Preserve each parameter's vLLM `weight_loader`; perform checkpoint layout
+conversion before invoking it. Account for every QKV/gate-up source shard.
+Keep precision-sensitive norm/modulation/embedders in BF16 unless separately
+proven, using the current `safe_quant_config` pattern where applicable.
+
+Validate resident DiT FP8 first: prove which modules are quantized, HBM saved,
+BF16-vs-FP8 trajectory/artifact quality, latency, and throughput. Text encoder,
+VAE, other hardware, TP/cache/DLO combinations are separate rows.
+Pre-quantized, pruned, rotated, or adapter-modified checkpoints are separate
+model/loading lanes; compare them with the released BF16 model without calling
+the result a pure quantization ablation. Read
+[`../quantization/SKILL.md`](../quantization/SKILL.md) if present and prefer the
+target revision's implementation/docs when sibling guidance differs.
+
+### Gate 4: Make every component offloadable
+
+Declare topology instead of relying on heuristic discovery:
+
+```python
+from typing import ClassVar
+
+import torch.nn as nn
+
+from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
+from vllm_omni.diffusion.offloader import OffloadPlan
+
+class MyPipeline(nn.Module, SupportsComponentDiscovery):
+    _dit_modules: ClassVar[list[str]] = ["transformer"]
+    _encoder_modules: ClassVar[list[str]] = ["text_encoder"]
+    _vae_modules: ClassVar[list[str]] = ["vae"]
+    _resident_modules: ClassVar[list[str]] = []
+    _offload_plan: ClassVar[OffloadPlan] = OffloadPlan(
+        on_demand_component_paths=frozenset({"text_encoder", "vae"}),
+        block_attrs={"transformer": ("blocks",)},
+        offload_submodules={"token_refiner": "blocks"},
+        resident_dit_paths=frozenset({"transformer"}),
+        encoder_block_attrs={"text_encoder": ("encoder.layers",)},
+    )
+```
+
+Add a test that resolves every dotted component/block path and fails if it is
+missing or is not the expected module/container; discovery may otherwise warn
+and skip. Validate ordinary layerwise offload and distributed layerwise
+offload (DLO) independently.
+
+Keep component lifecycle and host-weight storage separate. `OffloadPlan`
+declares what can be streamed; the diffusion loader owns any `HostWeightPlan`
+and hands the exact prevalidated plan to DLO. Do not make the offloader rescan
+checkpoint files or duplicate loader name/shape/dtype decisions. Direct mmap is
+a fail-closed optimization: current preflight requires TP1, no HSDP, no online
+quantization, complete bindings, and represented transforms. Otherwise the
+ordinary loader remains authoritative.
+
+If the target revision supports Host Weight Runtime (HWR), treat it as a
+separate, exact-identity startup/cache path rather than generic offload or
+zero-copy execution. Qualify `preferred` population/fallback and `required`
+consume-only failure independently; verify immutable manifests, final-layout
+restore, lease cleanup, corruption/quarantine, and node-local sharing. Current
+model contracts may be BF16 no-AllGather-only, so reject AllGather, online
+quantization, HSDP, LoRA/adapted weights, and unrecognized load formats unless
+the exact target revision explicitly adds them.
+
+On small-HBM cards, record load/materialization peak, resident peak, encode,
+each denoise wave, decode, transient per-rank HBM, host PSS, H2D, and collective
+time. Test DLO AllGather and `--dlo-no-use-allgather` as different deployments.
+Sweep the supported resident-block policies and publish the non-dominated
+latency-HBM frontier; a single minimum-memory or fastest point is not the DLO
+trade-off. Keep request-specific preparation replica-local. Only weight
+materialization may enter the declared DLO collective group; never use WORLD
+for per-request work in a DP deployment.
+Direct checkpoint mmap preflight is limited to TP1 without HSDP or online
+quantization; TP>1 falls back to the ordinary TP-aware loader and can still feed
+DLO AllGather or no-AllGather after scoped E2E. HSDP remains incompatible with
+DLO AllGather. A target revision may allow finalized per-tensor online FP8
+through the ordinary loader and AllGather; keep every other online quantizer
+fail-closed, and require model/card/topology E2E before promotion.
+Use deploy YAML for DP under `--omni`; vLLM DP CLI flags are rejected.
+
+### Gate 5: Add request-scoped acceleration safely
+
+Server-wide Cache-DiT support does not imply a per-request quality API. Only
+add request policy after defining calibrated quality tiers and a safe hook
+lifecycle using `SupportsRequestScopedCacheDiT`, `CacheDiTRequestSpec`, and
+`RequestScopedCacheDiTRuntime`. Validate alternating and concurrent quality
+tiers, real cache hits, refresh/reset, success, error, disconnect, abort, and
+the next uncached request. Do not batch different quality policies unless the
+batch compatibility key and hook ownership make that safe.
+
+Then test Cache-DiT against FP8, DLO, TP/SP, compile, and batching one
+combination at a time. See [production feature patterns](references/feature-patterns.md)
+for the lifecycle skeleton and compatibility matrix.
+
+### Gate 6: Optimize only from profiles
+
+Freeze one canonical workload manifest before profiling. Keep lossless runtime
+A/Bs, accelerated paths with numerical/quality trade-offs, and production
+topology studies in separate result lanes. Reconcile the client boundary as
+queue + encode + denoise + video/audio decode + output transport/codec +
+residual; do not hide a material residual in another stage.
+
+Profile stage time, GPU kernels, CPU/GPU synchronization, allocations, H2D,
+output payload/copies, and collectives before choosing work. Apply and A/B one
+change at a time:
+
+1. Remove synchronization and redundant denoise-loop work: hoist static prompt
+   conditioning/token refinement, RoPE tables, masks/metadata, bounded AdaLN
+   schedule projections, and reference scans; remove per-layer `.item()` and
+   unused masks/allocations.
+2. Select the fastest correct BF16 attention backend for each role and shape.
+3. Prefer explicit fused QKV, gate-up + `SiluAndMul`, RMSNorm/RoPE, AdaLN, and
+   layout/residual operators with guarded native fallbacks. Preserve every
+   materialized dtype/rounding boundary consumed by the unfused graph.
+4. Enable sparse attention only when platform, shape, topology, realized
+   sparsity, quality, fallback, and end-to-end speedup are all proven.
+5. Profile USP all-to-all count/bytes, packing, contiguous copies, overlap, and
+   scaling efficiency. Audit replicated cross-attention before using
+   `skip_sequence_parallel`. Qualify regular and accelerated Ulysses transport
+   independently; activation logs, JIT/readiness time, workspace growth,
+   stream ownership, maximum-shape warmup, and numerical drift are gates.
+   Preserve the checkpoint's Q:KV head ratio when padding GQA for Ulysses, and
+   validate strided QKV staging rather than assuming packed contiguity.
+6. Evaluate text-encoder or VAE disaggregation only if stage share, transfer
+   volume, reuse/concurrency, and independent scaling justify it. Tiling,
+   offload, or patch parallelism is not disaggregation.
+
+Do not combine independent optimizations into one performance claim. Preserve
+raw before/after runs with fixed workload and warmup policy.
+
+### Gate 7: Make serving interruptible and load-stable
+
+Treat request-mode batching and step-wise execution as separate capabilities:
+
+- request batching requires `supports_request_batch = True` and
+  `DiffusionRequestBatch -> list[DiffusionOutput]`, exactly one output per
+  request;
+- step execution requires the complete `SupportsStepExecution` contract:
+  `prepare_encode`, `denoise_step`, `step_scheduler`, and `post_decode`.
+
+Keep all mutable scheduler, RNG, latent, mask, cache, and step state request
+scoped. First validate `--step-execution --max-num-seqs 1` and step-boundary
+abort; only then test heterogeneous multi-request waves. Step continuous
+batching is experimental and is not automatically compatible with Cache-DiT.
+Before recommending it, predeclare a useful case and success threshold, then
+A/B it against request mode under the same arrival process. Structural support
+or a generic bridge is not latency, throughput, or cancellation evidence for a
+model.
+
+Test queued and in-flight abort, client disconnect, timeout, OOM, worker error,
+and restart. Require one terminal result, idempotent cleanup, no post-decode
+after abort, no cache/temp/VRAM/request-ID leak, and a successful next request.
+Inject late worker results/exceptions after cancellation and prove the result
+pump stays alive instead of raising `InvalidStateError`. For large batched or
+long-window video outputs, freeze dtype, range, layout, contiguity, ownership,
+payload size, serializer/codec limits, and offline-versus-HTTP semantics.
+Validate device-side output preparation, D2H/IPC or shared handles, remote
+encoding, event-loop responsiveness, and client materialization as separate
+stages, including abort and consumer failure cleanup.
+A model/VAE callback that publishes ordered chunks is only a producer contract;
+it is not transport, backpressure, cancellation, public streaming, or
+time-to-first-frame evidence. Distinguish complete-response faster-than-playback
+(`client E2E / output duration <= 1`) from streaming and report first-chunk,
+steady cadence, finalization, and complete-artifact latency separately.
+Run below-, near-, and above-saturation mixed-RPS soaks and report success/error
+rate, throughput, queue time, memory slope, temp growth, and worker health.
+Report p50/p95/p99 only from a declared arrival model with enough measured
+requests for the claimed percentile. See
+[serving and CI validation](references/production-validation.md).
+
+### Gate 8: Publish recipes and four independent CI tracks
+
+For every target vendor/card/topology, publish exact environment, serve/deploy
+command, one complete JSON or multipart curl per supported task, expected
+output checks, benchmark command, warmup/repeat/concurrency policy, raw metrics,
+per-rank HBM/host PSS, accuracy artifact, and limitations. Mark each
+CUDA/ROCm/NPU/XPU row `validated`, `unsupported`, or `not tested`; never inherit
+support from the platform abstraction or another card. Re-run DLO on each
+small-HBM recipe.
+
+Maintain separate CI gates:
+
+1. Function: strict load, API/offline positive and negative contracts.
+2. Accuracy: fixed reference revision/assets/seed/schedule and scoped tolerance.
+3. Performance: fixed best deployment plus memory-constrained/DLO row, raw JSON,
+   explicit warmup, enough samples for each claimed statistic, and owned
+   regression thresholds.
+4. Reliability: long mixed-RPS soak, abort/disconnect/fault cleanup, memory and
+   temporary-resource trend.
+
+## Production Definition of Done
+
+A model is production-ready only when:
+
+- every official advertised task passes API parity, offline/online parity,
+  strict loading, fixed-reference quality, and negative validation;
+- the recommended deployment row passes correctness, isolation, abort/error
+  cleanup, benchmark, and soak gates on named hardware;
+- online FP8 and DLO are each implemented and either validated in scoped rows
+  or explicitly limited/unsupported with fail-fast behavior;
+- per-request cache quality tiers have measured speed/quality frontiers and
+  request isolation, if advertised;
+- CUDA/ROCm/NPU/XPU recipes describe evidence rather than inferred support;
+- Function, Accuracy, Performance, and Reliability CI have owners, artifacts,
+  and actionable failure output;
+- accelerated paths identify every precision/quality trade-off, and the raw
+  offline plus online encoded output contracts are versioned and tested;
+- all unverified task/backend/cache/quant/offload/topology/hardware combinations
+  remain `not tested` and are not presented as supported.
+
+Day-0 support is still a valid milestone, but title, docs, and matrix must say
+which representative task passed and which production gates remain.

.claude/skills/production-add-diffusion-model/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "Production Add Diffusion Model"
+  short_description: "Productionize vLLM-Omni diffusion models"
+  default_prompt: "Use $production-add-diffusion-model to take this integrated diffusion model from Day-0 support to evidence-backed production readiness."

.claude/skills/production-add-diffusion-model/references/api-and-recipes.md

@@ -0,0 +1,315 @@
+# API Contract and Hardware Recipes
+
+## Contents
+
+1. [Freeze the upstream contract](#freeze-the-upstream-contract)
+2. [Validate before side effects](#validate-before-side-effects)
+3. [Task request templates](#task-request-templates)
+4. [Serving benchmark template](#serving-benchmark-template)
+5. [Hardware recipe contract](#hardware-recipe-contract)
+6. [Best deployment selection](#best-deployment-selection)
+
+## Freeze the upstream contract
+
+Pin three revisions independently:
+
+- official source implementation commit or package version;
+- checkpoint/model-card revision and partition;
+- vLLM-Omni target commit.
+
+Build a table before editing code. One row is one official task/partition.
+
+| Field | Required content |
+|---|---|
+| Task and partition | Official task name, checkpoint subdirectory, task selection rules |
+| Call contract | Constructor and call arguments, types, required/optional, official defaults |
+| Prompt/input | Prompt count/length, modality, MIME, order, count, decoded shape |
+| Size/time | Width/height alignment, pixels, frames, duration, FPS, start offset |
+| Denoise | Scheduler, sigma schedule, steps, CFG/guidance, seed behavior |
+| Output | Count, format, codec/sample rate, return type |
+| API | Offline mapping, sync endpoint, async endpoint/job lifecycle |
+| Rejection | Status code and stable error class/message for every invalid boundary |
+
+Official API parity includes limits and defaults, not just names. Do not add a
+generic default that changes a distilled checkpoint's fixed schedule, or accept
+an input combination the official task rejects.
+
+For every accepted boundary, add the neighboring rejected values. Examples:
+`max_images` and `max_images + 1`, smallest/largest legal duration, aligned and
+misaligned dimensions, omitted and illegal steps, one and too many outputs.
+
+## Validate before side effects
+
+Use one normalization/validation path for offline, sync, and async requests.
+The ordering must be:
+
+```text
+parse -> normalize aliases -> validate task/source matrix -> validate declared
+limits -> stream with byte caps -> inspect/decode with frame/pixel/duration caps
+-> persist only if needed -> submit to engine
+```
+
+Fail before download, persistence, job creation, or engine submit whenever the
+available metadata is sufficient. For streamed uploads, enforce:
+
+- total reference count and per-modality count before reading bodies;
+- per-file bytes while streaming;
+- aggregate bytes across all files;
+- decoded frames, duration, pixels, and media type before engine submission.
+
+On rejection, close streams and remove every partial temporary resource. Use
+HTTP 400 for invalid task/field combinations and 413 for payload limits. Test
+both sync and async paths; an invalid async request must not create a job ID.
+
+Do not trust client MIME alone. Sniff/validate content and keep ordered repeated
+multipart fields ordered. Avoid logging prompts, raw references, signed URLs,
+or base64 payloads.
+
+## Task request templates
+
+These are endpoint-shape templates, not model recipes. Replace placeholders
+with the exact official defaults and validated assets for the target model.
+Every production recipe must include all advertised tasks and an expected
+output check.
+
+### Text to image: JSON
+
+```bash
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/images/generations \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "<model>",
+    "prompt": "<validated prompt>",
+    "size": "1024x1024",
+    "num_inference_steps": 30,
+    "seed": 42,
+    "response_format": "b64_json"
+  }' | jq -er '.data[0].b64_json' | base64 -d > output.png
+file output.png
+```
+
+### Image edit: multipart
+
+```bash
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/images/edits \
+  -F 'model=<model>' \
+  -F 'prompt=<validated edit prompt>' \
+  -F 'image=@/path/to/reference.png;type=image/png' \
+  -F 'size=1024x1024' \
+  -F 'num_inference_steps=30' \
+  -F 'seed=42' \
+  -F 'output_format=png' \
+  | jq -er '.data[0].b64_json' | base64 -d > edited.png
+file edited.png
+```
+
+Use repeated `image` fields only if the model's official input matrix allows
+multiple ordered sources. Never describe a file upload as JSON-only.
+
+### Public request fields versus internal sampling fields
+
+Use the field exposed by the target revision's endpoint schema; do not rename
+it from the pipeline's internal attribute. For example, the current video JSON
+and multipart contracts expose `extra_params`, while serving translates that
+object into `DiffusionSamplingParams.extra_args`. Other request paths may
+expose canonical `extra_args` or nest it under `extra_body`. Inspect the exact
+Pydantic model/FastAPI form dependency and exercise the generated curl before
+publishing a recipe. A deprecation warning in a different endpoint is not
+evidence that an unrecognized multipart form field works here.
+
+Current endpoint map (re-check it at the target commit):
+
+| Endpoint | Public model-specific field |
+|---|---|
+| `/v1/videos`, `/v1/videos/sync` | Multipart `extra_params` JSON string |
+| `/v1/images/generations` | JSON `extra_params` object |
+| `/v1/chat/completions` diffusion | `extra_body.extra_args` |
+| `/v1/images/edits` | Declared typed form fields only |
+| `/v1/audio/generate` | Declared typed JSON fields only |
+
+Prefer declared top-level fields. Use a generic model-specific envelope only
+when the endpoint actually exposes it.
+
+### Text to video: multipart
+
+```bash
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
+  -H 'Accept: video/mp4' \
+  -F 'model=<model>' \
+  -F 'prompt=<validated prompt>' \
+  -F 'size=1280x720' \
+  -F 'num_frames=81' \
+  -F 'fps=16' \
+  -F 'num_inference_steps=30' \
+  -F 'seed=42' \
+  -F 'extra_params={"task":"<official-task>"}' \
+  -o output.mp4
+ffprobe -v error -show_entries stream=codec_name,width,height,r_frame_rate \
+  -of json output.mp4
+```
+
+### Image/video to video: multipart
+
+```bash
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
+  -H 'Accept: video/mp4' \
+  -F 'model=<model>' \
+  -F 'prompt=<validated prompt>' \
+  -F 'num_frames=81' \
+  -F 'fps=16' \
+  -F 'num_inference_steps=30' \
+  -F 'seed=42' \
+  -F 'extra_params={"task":"<official-task>","start_time_seconds":0}' \
+  -F 'input_reference=@/path/to/reference.png;type=image/png' \
+  -o i2v.mp4
+
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
+  -H 'Accept: video/mp4' \
+  -F 'model=<model>' \
+  -F 'prompt=<validated prompt>' \
+  -F 'num_frames=81' \
+  -F 'num_inference_steps=30' \
+  -F 'seed=43' \
+  -F 'extra_params={"task":"<official-task>","start_time_seconds":0}' \
+  -F 'input_reference=@/path/to/reference.mp4;type=video/mp4' \
+  -o v2v.mp4
+```
+
+For multiple references, repeat the model's accepted multipart field in
+official order. Include complete downloadable or repository fixtures plus
+SHA256 hashes. A placeholder-only recipe is not runnable evidence.
+
+### Text to audio: JSON
+
+```bash
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/audio/generate \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "<model>",
+    "input": "<validated audio prompt>",
+    "audio_length": 10.0,
+    "num_inference_steps": 100,
+    "seed": 42,
+    "response_format": "wav"
+  }' -o output.wav
+ffprobe -v error -show_entries stream=codec_name,sample_rate,channels \
+  -of json output.wav
+```
+
+Use `/v1/audio/generate` for general diffusion audio. Do not substitute the
+speech API unless the model is actually served through the speech contract.
+
+### Async video lifecycle
+
+Alongside `/v1/videos/sync`, include `POST /v1/videos`, status polling,
+content retrieval, cancellation/abort if public, and cleanup assertions. The
+same request must normalize to the same engine sampling parameters in both
+modes.
+
+## Serving benchmark template
+
+Run a smoke first, then fixed-concurrency and arrival-rate tests. Video task
+names for the harness are `t2v`, `i2v`, `ti2v`, or `v2v`; model-specific task
+selection belongs in `--extra-body` when required.
+
+```bash
+python benchmarks/diffusion/diffusion_benchmark_serving.py \
+  --base-url http://127.0.0.1:8091 \
+  --endpoint /v1/videos \
+  --model '<model>' \
+  --dataset random \
+  --task t2v \
+  --num-prompts 100 \
+  --width 1280 --height 720 \
+  --num-frames 81 --fps 16 \
+  --num-inference-steps 30 \
+  --request-rate 0.2 \
+  --max-concurrency 4 \
+  --warmup-requests 4 \
+  --warmup-concurrency 4 \
+  --extra-body '{"extra_params":"{\"task\":\"<official-task>\"}"}' \
+  --output-file result.json
+```
+
+The video backend currently stringifies each multipart value. Pre-serialize
+the nested `extra_params` object as above; passing it as a nested object would
+produce Python dict syntax rather than valid JSON. If the target revision
+changes the backend to JSON-encode dict/list values, update and smoke-test this
+command against that revision.
+
+For i2v/v2v, use the harness's validated dataset/trace path or provide the
+documented media source; do not silently benchmark t2v and label it i2v. Keep
+prompt, assets, seed, dimensions, frames, steps, guidance, warmup, concurrency,
+and arrival process identical across A/B runs.
+
+Keep three benchmark lanes distinct:
+
+1. lossless runtime A/B with the released schedule/precision/attention policy;
+2. accelerated-path A/B that changes one adapter, quantization, cache, or
+   sparsity policy and adds same-seed quality evidence;
+3. production-topology studies that vary placement or parallelism and report a
+   latency/throughput/memory frontier rather than a kernel speedup.
+
+For a fixed single-request A/B, retain every measured run and report median or
+mean with range after an explicit warmup; do not infer p95/p99 from a few
+repetitions. Tail latency requires a declared arrival process, request count,
+concurrency, and enough successful samples. Reconcile client E2E with queue,
+encoder, denoise, video/audio decode, D2H/IPC, codec, and residual timings.
+
+For generated media, define `real-time` rather than relying on the label. A
+complete-response claim uses `client E2E / validated output duration <= 1` and
+does not imply streaming or low time-to-first-frame. Report first chunk/fragment
+and cadence separately when a public streaming route exists. Never add gains
+across benchmark lanes with different schedules, prompts, seeds, artifacts, or
+topologies.
+
+The diffusion serving harness currently covers image/video task families, not
+`/v1/audio/generate`. For text-to-audio, use the target model's task-specific
+audio benchmark if one exists; otherwise add a serving harness that records
+request rate, concurrency, latency, generated-audio seconds per second, error
+rate, and output validation. Do not relabel the TTS harness or a video workload
+as diffusion-audio evidence.
+
+## Hardware recipe contract
+
+Create a separate row and runnable section for each vendor/card/topology:
+
+| Evidence | Required detail |
+|---|---|
+| Identity | Vendor, exact SKU, device count, HBM, interconnect |
+| Software | Driver, CUDA/ROCm/CANN/oneAPI, framework/container, dependency versions |
+| Source | vLLM-Omni commit, official/checkpoint revision and hashes |
+| Deployment | dtype, attention backend per role, TP/SP/CFG/HSDP/DP, cache, quant, offload |
+| Commands | Complete deploy YAML/serve command, every task curl, benchmark command |
+| Method | Warmup exclusion, repetitions or request count, concurrency, arrival distribution, duration, statistic definitions |
+| Results | Success rate, p50/p95/p99, RPS/throughput per device, stage latency |
+| Memory | Load/materialize, resident, encode, denoise, decode, transient per-rank HBM and host PSS |
+| Quality | Metric/tolerance, artifact links/hashes, temporal/audio checks |
+| Output | Raw/offline dtype, range, layout and ownership; online codec, payload bytes, transport route and client boundary |
+| Limits | `validated`/`limited`/`unsupported`/`not tested`, fallback/rejection behavior |
+
+Use current paths such as `recipes/<vendor>/`, `recipes/README.md`, supported
+models, and feature compatibility tables. Top-level documentation must link to
+the scoped matrix; it must not use an unqualified checkmark that hides a ROCm,
+NPU, XPU, task, topology, or dtype limitation.
+
+DLO evidence is per card, especially on small-HBM devices. A DLO result on one
+CUDA card does not establish another CUDA SKU, ROCm, NPU, or XPU support.
+
+## Best deployment selection
+
+Benchmark a correctness-approved candidate matrix, not every theoretical
+combination. Choose the recommended row by the production objective—latency,
+throughput, cost, or small-HBM fit—and document that objective.
+
+Do not recommend a row until it passes:
+
+- all advertised task/API contracts;
+- fixed-reference quality;
+- request isolation and abort/error cleanup;
+- sustained arrival-rate tests;
+- per-rank HBM and host PSS limits;
+- hardware-specific accuracy and performance CI.
+
+Keep the most memory-constrained DLO row even when it is not the fastest; it is
+a separate production recipe, not a failed latency candidate.

.claude/skills/production-add-diffusion-model/references/feature-patterns.md

@@ -0,0 +1,462 @@
+# Production Feature Patterns
+
+## Contents
+
+1. [Compatibility matrix](#compatibility-matrix)
+2. [Online FP8](#online-fp8)
+3. [Strict fused-weight loading](#strict-fused-weight-loading)
+4. [Distributed layerwise offload](#distributed-layerwise-offload)
+5. [Per-request Cache-DiT](#per-request-cache-dit)
+6. [Combination gates](#combination-gates)
+
+## Compatibility matrix
+
+Create rows at the granularity below. Add evidence links and a reason for every
+non-validated state.
+
+| Task/shape | Execution mode/capacity | Backend/layout | Quant | Cache policy | Offload | Topology | Hardware/dtype | Output contract/transport | State | Evidence |
+|---|---|---|---|---|---|---|---|---|---|---|
+| `<task>/<shape>/<schedule>` | serial request / 1 | SDPA, padded | BF16 | none | resident | 1 device | `<card>` BF16 | raw float / local | `not tested` | — |
+| `<task>/<shape>/<schedule>` | step / N | `<fast>`, packed | BF16 | none | resident | `<TP/SP>` | `<card>` BF16 | encoded / subprocess | `not tested` | — |
+| `<task>/<shape>/<schedule>` | step / N | `<fast>`, packed | FP8 | high | DLO no-AG | `<TP/SP>` | `<card>` | device uint8 / SHM | `not tested` | — |
+
+Initialize every row as `not tested`; the table is a work queue, not evidence.
+Promote the dense BF16 row to the oracle only after its scoped parity passes.
+Add one axis at a time. If the runtime cannot
+enforce a known-incompatible combination, add construction/admission
+validation before writing a recipe.
+
+## Online FP8
+
+### Component routing
+
+The public builder accepts one method or a longest-prefix component map:
+
+```python
+from vllm_omni.quantization import build_quant_config
+
+config = build_quant_config({
+    "transformer": {
+        "method": "fp8",
+        "ignored_layers": ["transformer.final_layer"],
+    },
+    "text_encoder": None,
+    "vae": None,
+    "default": None,
+})
+
+assert config.resolve("transformer.blocks.0.attn.qkv_proj") is not None
+assert config.resolve("text_encoder.layers.0.mlp") is None
+assert config.resolve("vae.decoder.conv_out") is None
+```
+
+Use actual runtime prefixes. Multi-DiT or nested pipelines may not use the
+generic names `transformer` and `vae`. A prefix that falls through to a default
+is not component validation. Add tests for every included and ignored prefix.
+
+For CLI serving, first validate resident DiT-only FP8:
+
+```bash
+vllm serve '<model>' --omni --quantization fp8 --max-num-seqs 1
+```
+
+For scoped routing:
+
+```bash
+vllm serve '<model>' --omni \
+  --diffusion-quantization-config \
+  '{"<dit-runtime-prefix>":{"method":"fp8","ignored_layers":["<exact-linear-prefix>"]},"<vae-runtime-prefix>":null,"default":null}' \
+  --max-num-seqs 1
+```
+
+Do not publish these placeholders. A recipe must contain the target model's
+verified prefixes.
+
+### Quantizable module wiring
+
+Pass `quant_config` and a stable prefix through every vLLM linear. Keep
+precision-sensitive modulation/norm layers safe unless proven otherwise.
+
+```python
+from vllm.model_executor.layers.activation import SiluAndMul
+from vllm.model_executor.layers.linear import (
+    MergedColumnParallelLinear,
+    QKVParallelLinear,
+    RowParallelLinear,
+)
+
+self.qkv_proj = QKVParallelLinear(
+    hidden_size=hidden_size,
+    head_size=head_dim,
+    total_num_heads=num_heads,
+    total_num_kv_heads=num_kv_heads,
+    bias=False,
+    quant_config=quant_config,
+    prefix=f"{prefix}.attn.qkv_proj",
+)
+self.gate_up_proj = MergedColumnParallelLinear(
+    hidden_size,
+    [intermediate_size, intermediate_size],
+    bias=False,
+    quant_config=quant_config,
+    prefix=f"{prefix}.mlp.gate_up_proj",
+)
+self.act_fn = SiluAndMul()
+self.down_proj = RowParallelLinear(
+    intermediate_size,
+    hidden_size,
+    bias=False,
+    input_is_parallel=True,
+    quant_config=quant_config,
+    prefix=f"{prefix}.mlp.down_proj",
+)
+```
+
+Set required checkpoint parameters to fail rather than silently initialize if
+the current loader supports `missing_param_init = "error"`.
+
+### FP8 evidence
+
+For each task/hardware row, capture:
+
+- startup proves the FP8 quant method is attached to intended linear modules;
+- no unexpected BF16 fallback and no meta/uninitialized parameters;
+- same-seed component, trajectory, and final-artifact comparison with BF16;
+- HBM at load, resident, encode, denoise, decode, and peak;
+- cold load time; fixed-work warm-latency raw runs; and serving p50/p95/p99 plus
+  throughput only from a declared arrival load with enough samples;
+- named ignored layers and why their BF16 retention is required.
+
+DiT FP8 evidence does not cover the text encoder or VAE. CUDA evidence does not
+cover ROCm/NPU/XPU. A pre-quantized checkpoint is a different loading path from
+online FP8 and needs its own state. If it is also pruned, rotated, distilled, or
+adapter-modified, report that model delta explicitly; BF16-versus-artifact
+quality and speed are deployment comparisons, not pure quantization ablations.
+
+## Strict fused-weight loading
+
+Never use `param.data.copy_()` for quantizable vLLM parameters. Transform the
+checkpoint layout before invoking the parameter loader so the online quant
+loader remains outermost:
+
+```python
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
+
+param = params[target_name]
+weight_loader = getattr(param, "weight_loader", default_weight_loader)
+
+if target_name.endswith(".qkv_proj.weight"):
+    packed_qkv = reorder_checkpoint_qkv(checkpoint_weight)
+    weight_loader(param, packed_qkv)
+elif target_name.endswith(".gate_up_proj.weight"):
+    gate, up = checkpoint_weight.chunk(2, dim=0)
+    weight_loader(param, gate, 0)
+    weight_loader(param, up, 1)
+else:
+    weight_loader(param, checkpoint_weight)
+```
+
+Track source shards per fused destination:
+
+```python
+expected = {
+    "blocks.0.attn.qkv_proj.weight": {"q", "k", "v"},
+    "blocks.0.mlp.gate_up_proj.weight": {0, 1},
+}
+seen = {name: set() for name in expected}
+
+# After mapping/loading each source:
+seen[target_name].add(shard_id)
+
+incomplete = {
+    name: sorted(shards - seen[name], key=str)
+    for name, shards in expected.items()
+    if shards - seen[name]
+}
+if incomplete:
+    raise RuntimeError(f"incomplete fused checkpoint parameters: {incomplete}")
+```
+
+Also compare retained model parameters with the returned loaded set. Explicit
+checkpoint ignores need a reason and test; an unknown or missing weight is not
+an ignore policy.
+
+Load-time adapter fusion creates a dedicated model, not a request-switchable
+LoRA. Pin the base and adapter revisions, validate every low-rank and full-rank
+delta target, freeze the adapter-declared schedule/task, and fuse before
+sharding only when the loader contract proves identical rank-local weights.
+Reject offload or host-weight paths that bypass that fusion rather than serving
+the unfused base model silently.
+
+## Distributed layerwise offload
+
+### Declare component topology
+
+Use `SupportsComponentDiscovery` and `OffloadPlan`. Validate all dotted paths
+at construction/test time because discovery may otherwise warn and skip.
+
+Treat a future/requested `--layerwise-offload-components` selector as a public
+API only if it exists in the target revision. Today, express component topology
+with `OffloadPlan` (`on_demand_component_paths`, `block_attrs`,
+`encoder_block_attrs`, `resident_dit_paths`, and `offload_submodules`). Do not
+invent a CLI flag or claim component-selective DLO from discovery declarations
+alone; validate the selector separately when it lands.
+
+```python
+from operator import attrgetter
+
+import torch
+
+def assert_offload_path(root, path):
+    value = attrgetter(path)(root)
+    if not isinstance(value, torch.nn.Module):
+        raise TypeError(f"offload path {path!r} is not an nn.Module")
+    return value
+
+for path in (
+    *pipeline._dit_modules,
+    *pipeline._encoder_modules,
+    *pipeline._vae_modules,
+    *pipeline._offload_plan.on_demand_component_paths,
+):
+    assert_offload_path(pipeline, path)
+```
+
+For `block_attrs`, resolve the DiT first and then each declared block attribute.
+Require an indexable block container with at least one `nn.Module`. Validate
+`offload_submodules`, `resident_dit_paths`, and `encoder_block_attrs` similarly.
+
+### Keep host storage loader-owned
+
+Current DLO accepts a loader-produced `HostWeightPlan`. Pipeline authors should
+not instantiate this plan or add ad hoc mmap capability flags. The loader must
+preflight complete runtime/checkpoint bindings, names, shapes, dtypes, persistent
+buffers, and represented transforms before it skips ordinary materialization.
+The exact accepted plan is consumed once by DLO; a rejected plan records an
+observable fallback reason and continues through the ordinary loader.
+
+Direct checkpoint mmap currently requires TP1 without HSDP or online
+quantization. TP>1 falls back to the ordinary TP-aware loader and may feed DLO
+AllGather or no-AllGather; the current design records a bounded DP2xTP2
+AllGather smoke, but that path does not receive direct-mmap host-page sharing.
+Per-tensor online FP8 must use the ordinary loader. Current revisions can admit
+its finalized weight/scale layout to DLO AllGather, including transposed runtime
+weights, while unsupported online quantizers still fail closed. Direct mmap
+remains unavailable, so capture the temporary full-model host materialization
+before rank sharding.
+
+Host Weight Runtime is a distinct opt-in path. Current final-layout consumers
+use no-AllGather DLO and exact model-declared BF16 representations. In
+`preferred` mode, consume an exact local hit or canonically load and publish for
+a future startup; in `required` mode, consume only and fail on a miss or unusable
+artifact. HWR must not interact with DLO AllGather, online quantization, HSDP,
+LoRA/adapted weights, or an unrecognized load format unless the target revision
+adds an exact producer/restore contract. It is immutable node-local CPU backing,
+not zero-copy GPU execution; the DLO transport still owns staging or direct H2D.
+Qualify identity inputs, cold population, warm reuse, atomic concurrent
+publication, source digests, corruption/quarantine, lease cleanup, capacity,
+node-local PSS, H2D, and E2E latency. A host-memory saving that materially
+regresses transfer or request latency is a separate memory-first deployment.
+
+### DLO deployment modes
+
+Treat these paths separately:
+
+| Path | Weight/runtime behavior | Required boundaries |
+|---|---|---|
+| DLO AllGather | Rank-sharded pinned host tensors; reconstruct layer through collective | TP>1 uses ordinary TP-aware loader output; HSDP and unsupported online quantizers are rejected; finalized per-tensor online FP8 is eligible only when recognized by the target revision; concurrent ranks must participate consistently |
+| DLO no-AllGather | Loader-approved checkpoint mmap or ordinary runtime tensors; each rank streams a complete block | Direct mmap is TP1/non-HSDP/non-online-quant only; TP/HSDP/online quant use ordinary runtime tensors and require scoped E2E |
+| SP + DLO | SP group can supply the DLO collective group | Packed boundaries and collective order require E2E |
+
+Single-stage examples:
+
+```bash
+# SP + DLO AllGather candidate; validate exact model/card before publishing.
+vllm serve '<model>' --omni \
+  --enable-distributed-layerwise-offload \
+  --usp 4 \
+  --max-num-seqs 1
+
+# TP + DLO rank-local candidate; not a support statement.
+vllm serve '<model>' --omni \
+  --tensor-parallel-size 2 \
+  --enable-distributed-layerwise-offload \
+  --dlo-no-use-allgather \
+  --max-num-seqs 1
+```
+
+Under `--omni`, configure DP in a deploy YAML rather than passing vLLM DP CLI
+flags:
+
+```yaml
+pipeline: <registered-pipeline>
+async_chunk: false
+data_parallel_size: 4
+stages:
+  - stage_id: 0
+    devices: "0,1,2,3"
+    max_num_seqs: 1
+    enable_distributed_layerwise_offload: true
+    dlo_use_allgather: true
+    parallel_config:
+      tensor_parallel_size: 1
+      sequence_parallel_size: 1
+```
+
+```bash
+vllm serve '<model>' --omni --deploy-config /path/to/dlo_dp4.yaml
+```
+
+Keep the YAML next to the validated recipe. Match its schema to the target
+revision; do not assume a model accepts another pipeline's deploy config.
+
+### DLO validation sequence
+
+1. Resident BF16 reference.
+2. Ordinary layerwise offload parity.
+3. DLO enable, warmup, same-seed generation, disable/restart.
+4. AllGather and no-AllGather separately.
+5. Concurrent requests with different prompts and identical collective-affecting
+   parameters; then reject or schedule incompatible steps/shapes safely.
+6. Queued/in-flight abort, error, idle rank, worker exit, and next request.
+7. TP, SP, cache, compile, online FP8 one axis at a time.
+8. Loader-plan selection/fallback reason, cold/warm storage lifecycle, and
+   corruption/stale-entry recovery when mmap/cache backing is advertised.
+9. Replica-local request preparation plus DLO-group-only weight collectives in
+   DP; exercise concurrent replicas to catch accidental WORLD rendezvous.
+10. Sweep resident block counts; publish raw latency/HBM pairs, dominated
+    points, and the non-dominated Pareto frontier.
+11. Per-rank HBM, host PSS for the full process tree, H2D and collective trace.
+
+On a small-HBM card, measure transient encode/VAE peaks as well as resident DiT
+weights. If a non-DiT component still exceeds HBM, combine only independently
+validated on-demand component staging, VAE tiling/patching, or another topology;
+do not hide the remaining peak behind the phrase “DLO enabled.”
+
+## Per-request Cache-DiT
+
+### Policy and lifecycle skeleton
+
+First verify that the target revision contains the shared protocol/runtime.
+If not, follow the nearest current model's lifecycle without inventing a new
+public API.
+
+The following is a **serial/exclusive request-mode sketch only**. Pipeline-wide
+hooks are mutable; concurrent requests need an admission/batching key and
+runner-owned transition serialization before this shape is safe.
+
+```python
+import torch.nn as nn
+
+from vllm_omni.diffusion.cache.cachedit import (
+    CacheDiTBackend,
+    CacheDiTRequestSpec,
+    RequestScopedCacheDiTRuntime,
+)
+
+class MyPipeline(nn.Module):
+    def __init__(self, *args, **kwargs):
+        super().__init__()
+        # ... components ...
+        self._cache_dit_runtime = RequestScopedCacheDiTRuntime(self)
+
+    def adopt_cache_dit_backend(self, backend: CacheDiTBackend) -> None:
+        self._cache_dit_runtime.adopt(
+            backend,
+            installation_key="my_model.generic",
+        )
+
+    def is_cache_dit_enabled(self) -> bool:
+        return self._cache_dit_runtime.is_enabled
+
+    def _cache_spec(self, quality, steps):
+        if quality == "lossless":
+            return None
+        if quality == "high":
+            return CacheDiTRequestSpec(
+                installation_key="my_model.high.v1",
+                cache_config=self._validated_high_config,
+                num_inference_steps=steps,
+            )
+        raise ValueError(f"unsupported quality tier: {quality!r}")
+
+    def forward(self, req):
+        quality, steps = validate_and_resolve_request(req)
+        self._cache_dit_runtime.prepare(self._cache_spec(quality, steps))
+        return self._run_validated_request(req)
+```
+
+The snippet shows the model-owned transition for exclusive execution. Put
+disable/restore handling in the runtime/runner lifecycle that owns success,
+exception, disconnect, and asynchronous abort; a `try/finally` inside
+`forward()` alone cannot observe every server cancellation boundary.
+
+The `installation_key` must include every policy dimension that changes hook
+installation; changes in step count can use runtime refresh. `None` disables
+installed hooks. Do not mutate the global startup cache config for a request.
+
+Centralize cleanup in the owning request/runner boundary. An exception, abort,
+disconnect, or admission failure must not leave a changed hook policy visible
+to the next request. The exact owner depends on the target revision; add a
+failure-injection test rather than relying on `forward()` alone to observe
+asynchronous abort.
+
+Example model-specific request form after calibration:
+
+```bash
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
+  -F 'model=<model>' \
+  -F 'prompt=<prompt>' \
+  -F 'quality=lossless' \
+  -F 'num_inference_steps=<validated-steps>' \
+  -o lossless.mp4
+
+curl --fail-with-body -sS -X POST http://127.0.0.1:8091/v1/videos/sync \
+  -F 'model=<model>' \
+  -F 'prompt=<prompt>' \
+  -F 'quality=high' \
+  -F 'num_inference_steps=<validated-steps>' \
+  -o high.mp4
+```
+
+Do not expose these tier names merely because another model uses them.
+
+### Cache-DiT evidence
+
+For every tier/task/shape/schedule:
+
+- show real cache hits/skipped blocks, not only “backend enabled” logs;
+- measure fixed-work speed and quality against lossless/native;
+- publish a speed/quality frontier and chosen tier mapping;
+- alternate lossless/high/lossless and vary steps to prove restore/refresh;
+- interleave tasks and shapes; run concurrent requests only when scheduling
+  guarantees exclusive compatible hook state;
+- inject validation error, denoise error, disconnect, and abort, then prove the
+  next request starts with its requested policy;
+- bound any cached tensors and clear request-local state.
+
+## Combination gates
+
+Start every combination as `not tested`.
+
+| Combination | Default production posture |
+|---|---|
+| Direct checkpoint mmap + TP>1/HSDP/online quant | Preflight falls back to the ordinary loader; do not claim direct-mmap savings |
+| Per-tensor online FP8 + DLO AllGather | Generic compatibility exists for finalized weights/scales; each model/card/topology remains a candidate until E2E, and other online quantizers fail closed |
+| Online FP8 + DLO no-AllGather | Candidate only after model/card/topology E2E |
+| TP>1 + DLO AllGather | Ordinary TP-aware loader only; bounded DP2xTP2 smoke exists, but each new model/card/topology still needs E2E |
+| TP>1 + DLO no-AllGather | Ordinary TP-aware loader only; candidate with limited generic coverage |
+| HSDP + DLO AllGather | Reject; it would double-shard HSDP-managed parameters |
+| HSDP + DLO no-AllGather | Configuration-compatible candidate with limited E2E coverage |
+| Cache-DiT + online FP8 | Separate trajectory, hit-rate, HBM, latency test |
+| Cache-DiT + DLO | Verify hooks, streamed blocks, memory peaks, abort cleanup |
+| Cache-DiT + step execution | Do not advertise until lifecycle/batching support is explicit |
+| Sparse attention + Ring | Verify backend execution; reject silently ignored sparse settings |
+| Load-time fused adapter + offload/HWR | Reject unless the selected loader path applies and validates the same fusion before sharding/storage |
+| Packed attention + any topology | Unequal-sample boundary parity is mandatory |
+| Device-side uint8 preparation + offline caller | Version the dtype/range/layout change; HTTP MP4 parity does not preserve the raw tensor contract |
+| Device-side preparation + remote codec | Validate route selection, strided/contiguous layouts, payload bytes, ordering, fallback, and encoded-media parity |
+
+A startup-only test never upgrades a combination. Require request output,
+parity/quality, actual fast-path evidence, resource cleanup, and a benchmark on
+the named hardware.

.claude/skills/production-add-diffusion-model/references/performance-patterns.md

@@ -0,0 +1,481 @@
+# Performance Patterns and Code Examples
+
+## Contents
+
+1. [Measurement loop](#measurement-loop)
+2. [Faster BF16 attention](#faster-bf16-attention)
+3. [Sparse attention](#sparse-attention)
+4. [Fusion-first operator patterns](#fusion-first-operator-patterns)
+5. [Remove redundant computation](#remove-redundant-computation)
+6. [USP communication optimization](#usp-communication-optimization)
+7. [Text encoder and VAE disaggregation](#text-encoder-and-vae-disaggregation)
+8. [Performance acceptance](#performance-acceptance)
+
+## Measurement loop
+
+Do not start with a list of optimizations. Start with a fixed correctness and
+performance workload:
+
+```text
+model/checkpoint revision + prompt/media hashes + seed + dimensions/frames +
+scheduler/sigma/steps/guidance + task + dtype + backend + topology + hardware
+```
+
+Capture:
+
+- stage time: load, encode, conditioning, denoise, VAE decode/postprocess;
+- GPU trace: kernel duration/count, launch gaps, allocations, H2D, collectives;
+- CPU trace: preprocessing, per-layer sync, Python dispatch, temporary files;
+- memory: per-rank HBM peaks and process-tree host PSS;
+- output: device preparation, D2H/IPC, payload bytes, codec wall/process CPU,
+  event-loop wait, client materialization, and signed residual;
+- serving: queue time, throughput, errors, request mix, and distribution samples.
+
+For a fixed-work single-request A/B, run one explicit warmup policy and at
+least three measured repetitions, keep every raw run, and report median or mean
+with range. Do not derive p95/p99 from three samples. For tail latency, declare
+the arrival model and collect enough successful requests for the percentile and
+confidence claimed. Implement one optimization at a time, re-run parity, and
+report both local kernel and end-to-end deltas. A kernel win that moves no E2E
+metric is not a production speed claim.
+
+Use the pipeline profiler already exposed by the target revision when
+available, and use PyTorch/CUDA/ROCm/NPU/XPU profiling tools appropriate to the
+platform. Do not force every platform through a CUDA-only workflow.
+
+## Faster BF16 attention
+
+### Shared attention wiring
+
+Use the shared role-aware attention layer and give it the true local head
+counts and QKV layout:
+
+```python
+import torch
+
+from vllm.model_executor.layers.linear import (
+    QKVParallelLinear,
+    RowParallelLinear,
+)
+from vllm_omni.diffusion.attention.layer import Attention
+
+class SelfAttention(torch.nn.Module):
+    def __init__(
+        self,
+        hidden_size,
+        head_dim,
+        num_heads,
+        num_kv_heads,
+        quant_config=None,
+        prefix="",
+    ):
+        super().__init__()
+        self.head_dim = head_dim
+        self.qkv_proj = QKVParallelLinear(
+            hidden_size=hidden_size,
+            head_size=head_dim,
+            total_num_heads=num_heads,
+            total_num_kv_heads=num_kv_heads,
+            bias=False,
+            quant_config=quant_config,
+            prefix=f"{prefix}.qkv_proj",
+        )
+        self.attention = Attention(
+            num_heads=self.qkv_proj.num_heads,
+            num_kv_heads=self.qkv_proj.num_kv_heads,
+            head_size=head_dim,
+            softmax_scale=head_dim**-0.5,
+            causal=False,
+            qkv_layout="BSND",
+            role="self",
+            prefix=prefix,
+        )
+        self.out_proj = RowParallelLinear(
+            num_heads * head_dim,
+            hidden_size,
+            bias=False,
+            input_is_parallel=True,
+            quant_config=quant_config,
+            prefix=f"{prefix}.out_proj",
+        )
+
+    def forward(self, x, attn_metadata=None):
+        qkv, _ = self.qkv_proj(x)
+        q_size = self.qkv_proj.num_heads * self.head_dim
+        kv_size = self.qkv_proj.num_kv_heads * self.head_dim
+        q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
+        q = q.unflatten(-1, (self.qkv_proj.num_heads, self.head_dim))
+        k = k.unflatten(-1, (self.qkv_proj.num_kv_heads, self.head_dim))
+        v = v.unflatten(-1, (self.qkv_proj.num_kv_heads, self.head_dim))
+        out = self.attention(q, k, v, attn_metadata)
+        out = out.flatten(-2)
+        out, _ = self.out_proj(out)
+        return out
+```
+
+Add Q/K norm and RoPE in the official order before the `Attention` call. Keep
+checkpoint mapping for the fused QKV and output projection complete.
+
+### Backend qualification
+
+For each role (`self`, cross-attention category, token refiner, etc.), record:
+
+- actual selected backend from logs/trace;
+- supported dtype, head size, QKV layout, mask/metadata, packed varlen behavior;
+- platform and compute capability;
+- TP/SP/Ring/AllGather-KV restrictions;
+- fallback behavior and whether fallback changes the claimed performance.
+
+Use unequal packed samples and compare with dense SDPA. Verify `cu_seqlens`,
+max sequence length, padding exclusion, modality boundaries, CFG ownership, and
+output slicing. Do not pass an `attn_mask` and assume it is consumed. Inspect
+the backend contract and trace.
+
+TRTLLM, Flash Attention, cuDNN, NPU varlen, and other backends are independent
+qualification rows. The shared `Attention` class does not make every backend
+packed-safe or faster automatically. Ring and hybrid paths require their own
+boundary parity. AllGather-KV and specific backend combinations may fail fast;
+keep that behavior visible in docs.
+
+## Sparse attention
+
+Sparse attention is a backend/model/platform optimization, not a synonym for
+low-precision dense attention. Enable it only after dense BF16 parity.
+
+For every candidate task/shape/topology:
+
+1. State the sparsity policy, selected backend/kernel, block/head dimensions,
+   supported hardware, external kernel/package revision, and fallback.
+2. Prove the sparse kernel executed and report realized sparsity; an enabled
+   flag is insufficient.
+3. Compare same-seed per-block outputs, denoise trajectory, and final artifact
+   with dense attention.
+4. Add modality-specific metrics: temporal consistency and audio/video sync for
+   video, spectral/audio quality for audio, and prompt/image metrics as relevant.
+5. Report attention time and full E2E time. Include preprocessing/index-build
+   overhead, peak memory, and the exact dense control.
+6. Validate each task and shape. Do not extrapolate a T2V result to I2V/Ref2V,
+   or NPU sparse support to CUDA/ROCm/XPU.
+
+If Ring bypasses the selected backend or a sparse setting would be silently
+ignored, reject the combination. Prefer a visible dense fallback only when the
+recipe labels the row `limited` and does not claim sparse speed.
+
+## Fusion-first operator patterns
+
+Prefer current shared/custom operators with explicit guards and native
+fallbacks. Prove parity at operator, block, trajectory, and artifact levels,
+then use a trace to prove fewer launches/bytes or lower latency.
+
+### Q/K RMSNorm + RoPE
+
+```python
+from vllm_omni.diffusion.layers.norm import RMSNorm
+from vllm_omni.diffusion.layers.rope import RotaryEmbedding, apply_rope_to_qk
+
+self.norm_q = RMSNorm(head_dim, eps=eps)
+self.norm_k = RMSNorm(head_dim, eps=eps)
+# Example only: set these booleans from the official checkpoint contract.
+use_neox_layout = False
+cos_sin_are_half_dim = True
+self.rope = RotaryEmbedding(
+    is_neox_style=use_neox_layout,
+    half_head_dim=cos_sin_are_half_dim,
+)
+
+q = self.norm_q(q)
+k = self.norm_k(k)
+q, k = apply_rope_to_qk(self.rope, q, k, (cos, sin))
+```
+
+Replace the illustrative booleans with the official layout in real code.
+Preserve partial rotary dimensions by splitting the rotated prefix and concatenating
+the pass-through suffix. If 3D cos/sin includes a batch axis, the current
+shared path assumes the values are shared across the batch; per-sample
+different positions require a proven representation/path.
+
+For packed `[tokens, heads, head_dim]` tensors with a non-interleaved
+`[cos | sin]` table, the current shared fused boundary is:
+
+```python
+from vllm_omni.diffusion.layers.fused_qk_norm_rope import fused_qk_norm_rope
+
+q, k = fused_qk_norm_rope(
+    q,
+    k,
+    self.norm_q.weight,
+    self.norm_k.weight,
+    rope_table,
+    self.norm_q.variance_epsilon,
+)
+```
+
+Its CUDA fast path currently specializes BF16 `head_dim=128` and
+`rotary_dim=96`; other valid inputs take the eager implementation. The separate
+shared RMSNorm and RotaryEmbedding still provide the general cross-platform
+baseline. Treat NPU/MUSA and other vendor fusions as independent qualification
+rows, and do not generalize a model-specific Triton kernel without compatible
+shape, layout, frequency partition, dtype, and platform evidence.
+
+### Fused QKV projection
+
+Use `QKVParallelLinear` as shown above. Qualification includes:
+
+- `num_heads` and `num_kv_heads` divisibility by TP;
+- local split sizes from the layer, never reusing global counts after TP;
+- Q/K/V checkpoint order conversion before the vLLM loader;
+- all Q, K, and V shards present;
+- quantization scale/loader mapping when FP8 or another method is enabled.
+
+### Gate-up projection + fused SwiGLU
+
+```python
+from vllm.model_executor.layers.activation import SiluAndMul
+from vllm.model_executor.layers.linear import (
+    MergedColumnParallelLinear,
+    RowParallelLinear,
+)
+
+self.gate_up_proj = MergedColumnParallelLinear(
+    input_size=hidden_size,
+    output_sizes=[intermediate_size, intermediate_size],
+    bias=False,
+    quant_config=quant_config,
+    prefix=f"{prefix}.gate_up_proj",
+)
+self.act_fn = SiluAndMul()
+self.down_proj = RowParallelLinear(
+    input_size=intermediate_size,
+    output_size=hidden_size,
+    bias=False,
+    input_is_parallel=True,
+    quant_config=quant_config,
+    prefix=f"{prefix}.down_proj",
+)
+
+gate_up, _ = self.gate_up_proj(x)
+x = self.act_fn(gate_up)
+x, _ = self.down_proj(x)
+```
+
+Map both checkpoint `gate_proj` and `up_proj` shards into the merged parameter
+and require both. A merged projection followed by `F.silu(gate) * up` is not
+the same claim as the `SiluAndMul` fused activation kernel; identify what the
+trace actually uses.
+
+### AdaLN
+
+Use a shared layer only when its mathematical/return contract matches the
+official model:
+
+```python
+from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
+
+self.adaln = AdaLayerNorm(
+    hidden_size,
+    elementwise_affine=False,
+    eps=1e-6,
+)
+x = self.adaln(x, scale[:, None, :], shift[:, None, :])
+```
+
+`AdaLayerNormZero`, `AdaLayerNormZeroSingle`, and continuous variants include
+different projections, broadcasts, and tuple outputs; they are not drop-in
+replacements. A shared AdaLN may have an NPU fast path while using native ops on
+CUDA/HIP/XPU. Do not call it a universal fused AdaLN kernel.
+
+For sigma schedules with a small finite set of timesteps, profile whether
+precomputing the timestep/AdaLN projection once per request is worthwhile.
+Cache keys must include the exact schedule/config/device/dtype and stay bounded.
+
+### Layout, residual, and other candidates
+
+After profiling, inspect:
+
+- bias/dropout/add or gated residual chains;
+- norm + linear or norm + modulation boundaries;
+- repeated transpose/reshape/contiguous around attention;
+- VAE normalization/activation/convolution boundaries;
+- MoE routing and fused experts for models that actually use MoE.
+
+For fused residual-plus-norm or modulation chains, preserve the unfused
+materialization boundary. If the residual was stored in BF16 before RMSNorm,
+round to BF16 and promote again inside the fused kernel; using the unrounded
+FP32 temporary changes the denoising trajectory even when the stored residual
+looks correct.
+
+Keep DLO materialize/release hooks, process-group collectives, request cache
+transitions, and cleanup outside compiled/fused regions unless lifecycle safety
+is explicitly tested. Compile only stable regions and retain eager fallback.
+
+## Remove redundant computation
+
+### Hoist request invariants
+
+Audit the denoise loop for work that depends only on the request rather than
+the current timestep:
+
+```python
+# Before the denoise loop.
+condition = self.condition_proj(text_embeddings)
+refined_condition = self.token_refiner(condition, packed_metadata)
+rope_tables = self.build_rope_tables(position_ids)
+attn_metadata = self.build_attention_metadata(layout)
+
+for step, sigma in enumerate(sigmas):
+    timestep_state = self.prepare_timestep_state(sigma)
+    noise = self.transformer(
+        latents,
+        refined_condition,
+        rope_tables=rope_tables,
+        attn_metadata=attn_metadata,
+        timestep_state=timestep_state,
+    )
+    latents = self.scheduler_step(noise, sigma, latents)
+```
+
+Candidates include prompt conditioning, token-refiner output, reference
+encoding/scans, position IDs/RoPE tables, packed masks/metadata, static CFG
+branches, and finite-schedule AdaLN projections. First prove they are invariant
+for all official tasks, LoRA/adapters, prompt-update features, shapes, and
+schedules. In continuous batching, store request-specific results on request
+state rather than the pipeline singleton.
+
+### Eliminate per-layer synchronization
+
+Avoid device-to-host scalar extraction such as `.item()` inside every layer:
+
+```python
+# Bad: forces a GPU/CPU synchronization in every attention layer.
+max_len = int(cu_seqlens[-1].item())
+
+# Better: compute validated host metadata once during request packing and pass it.
+packed = PackedLayout(
+    cu_seqlens=cu_seqlens,
+    max_seqlen=max(sample_lengths),
+)
+for block in self.blocks:
+    hidden = block(hidden, packed_layout=packed)
+```
+
+Trace to verify the synchronization disappears. Do not replace a correct
+dynamic value with a stale constant merely to remove `.item()`.
+
+### Remove unused masks and allocations
+
+If packed attention consumes `cu_seqlens` and ignores a dense mask, do not
+construct the dense mask. Reuse immutable request metadata and preallocated
+buffers when safe. Prefer direct indexed scatter/gather over constructing large
+temporary tensors only when duplicate-index semantics and gradients (normally
+disabled for inference) remain correct.
+
+Any cross-request cache needs a bounded lifetime and a complete key containing
+shape, layout, device, dtype, schedule, task, adapter/LoRA, and any value that
+changes the tensor. Otherwise keep it request-scoped.
+
+### VAE and reference work
+
+Profile duplicate VAE decode/postprocess, repeated reference decoding/scans,
+unnecessary host-device round trips, and full-frame materialization. Apply VAE
+tiling, slicing, patch parallelism, or on-demand staging only with seam,
+ordering, color/range, and memory parity tests.
+
+## USP communication optimization
+
+“USP works” is not evidence that USP communication is optimized. For Ulysses,
+Ring, hybrid, and AllGather-KV separately, profile:
+
+- collective count, bytes, duration, stream, and overlap with compute;
+- Q/K/V and output all-to-all boundaries;
+- pack/unpack, padding, metadata broadcast, transpose, and contiguous copies;
+- head divisibility or advanced uneven-head support;
+- scaling efficiency from 1 to N devices on the same workload/interconnect;
+- load balance for unequal packed samples.
+
+Declare `_sp_plan` split/gather boundaries around meaningful module outputs,
+not arbitrary Python statements. Keep packed metadata sharded/gathered in the
+same ownership model as tokens.
+
+When the target revision offers accelerated Ulysses transport such as
+`--ulysses-a2a-permute`, qualify it separately from regular Ulysses. Prove the
+strict scatter-heads/gather-sequence layout selected the fast path, record
+one-time JIT/readiness cost, prewarm the maximum workspace shape before CUDA
+graph capture, keep the grow-only workspace on one stream, and exercise cleanup
+before process-group destruction. Compare collective and E2E deltas after the
+dense backend and topology are frozen; keep unsupported layouts on an explicit
+regular-Ulysses fallback.
+
+For GQA, validate `Q_heads % KV_heads == 0`, pad KV heads first, and derive Q
+padding from the original ratio. Exercise both contiguous and row-strided QKV
+layouts; a fast copy path must preserve bitwise values and packed boundaries.
+
+Cross-attention sometimes uses replicated text K/V while only video/audio Q is
+sequence-sharded. In that specific case, `skip_sequence_parallel=True` can
+avoid incorrect or wasteful Q/K/V redistribution:
+
+```python
+self.cross_attention = Attention(
+    num_heads=local_heads,
+    num_kv_heads=local_kv_heads,
+    head_size=head_dim,
+    softmax_scale=head_dim**-0.5,
+    causal=False,
+    role="cross",
+    skip_sequence_parallel=True,
+    prefix=f"{prefix}.cross_attn",
+)
+```
+
+Use it only after proving K/V are replicated and Q/output ownership is correct.
+Otherwise it disables needed communication. Report whether the optimization
+reduces collective bytes/time and improves E2E scaling.
+
+## Text encoder and VAE disaggregation
+
+Disaggregation is a measured architecture decision. First collect:
+
+| Question | Required measurement |
+|---|---|
+| Is the stage large enough? | Encode/decode share at target concurrency and shape |
+| Is transfer economical? | Tensor/media bytes, serialization, connector latency and bandwidth |
+| Is there reuse? | Prompt/reference reuse rate, cacheability, fan-out |
+| Can it scale independently? | Queue saturation and resource utilization per stage |
+| Is the contract stable? | Shape, dtype, mask/tag/position metadata, ordering |
+
+### Text encoder candidate
+
+Define an explicit tensor contract containing hidden states, masks, tags or
+position IDs, dtype, shape, task/checkpoint revision, and request ID. Support
+disabling the local encoder so the DiT cannot accidentally recompute it. Test
+all tasks, prompt lengths, partial TP group membership, retries, backpressure,
+abort, connector cleanup, and independent replica scaling.
+
+### VAE candidate
+
+Separate VAE encode and decode contracts if both exist. Define latent shape,
+scale/shift, dtype/range, tiling/chunk order, output media ordering, and stage
+metadata. Measure latent/media transfer against local decode cost. Test tiling
+seams, RNG/determinism where relevant, chunk ordering, codec/postprocess,
+backpressure, abort, and worker failure.
+
+Do not label VAE tiling, patch parallelism, compile, or offload as VAE
+disaggregation. Likewise, a separate text-encoder TP group is not a separate
+stage. If no reusable generic stage/connector exists in the target revision,
+produce a feasibility result or RFC and keep the capability `not tested` rather
+than inventing a model-local production architecture.
+
+## Performance acceptance
+
+For each accepted optimization, require:
+
+- same fixed workload and correctness artifacts before/after;
+- actual fast-path execution proven by logs/trace/counters;
+- local metric plus E2E latency/throughput and memory;
+- at least three raw fixed-work repetitions after explicit warmup, or a
+  declared arrival-load sample set large enough for every tail percentile;
+- named hardware, backend, topology, dtype, and task scope;
+- fallback/rejection tests for unsupported shapes/platforms;
+- no regression in abort/error cleanup or long-run memory trend.
+
+Update the feature matrix narrowly. Do not convert a profile-driven direction
+into a generic “supported” feature merely because a model-specific PR exists.

.claude/skills/production-add-diffusion-model/references/production-validation.md

@@ -0,0 +1,464 @@
+# Production Serving and Validation
+
+## Contents
+
+1. [Execution-mode decision](#execution-mode-decision)
+2. [Request-level batching](#request-level-batching)
+3. [Step execution and continuous batching](#step-execution-and-continuous-batching)
+4. [Abort and cleanup](#abort-and-cleanup)
+5. [Mixed-RPS soak](#mixed-rps-soak)
+6. [Four CI tracks](#four-ci-tracks)
+7. [PR evidence checklist](#pr-evidence-checklist)
+
+## Execution-mode decision
+
+Request batching and step-wise execution solve different problems:
+
+| Mode | Pipeline contract | Scheduling/abort boundary |
+|---|---|---|
+| Serial request | `forward(DiffusionRequestBatch)` for one request | Whole request; in-denoise preemption is unavailable |
+| Request batching | `supports_request_batch = True`; one forward consumes compatible independent requests | Whole batch/request |
+| Single-request step | Full `SupportsStepExecution`; `max_num_seqs=1` | Between denoise steps |
+| Step continuous batching | Full step contract plus heterogeneous batched-step evidence | Between denoise waves/steps |
+
+Do not expose four stub methods as an opt-out. The current protocol is
+structural; if a native pipeline is not step-capable, do not define the four
+step methods. If it is capable, implement and test all four.
+
+Treat structural support, a generic request-batch bridge, and model benefit as
+three separate claims. Before recommending step execution, name a hypothesis
+that request mode cannot satisfy, define a success threshold, and run a matched
+request-mode control. If throughput or latency regresses, retain step execution
+as functional/limited and keep request mode as the production recommendation.
+
+Choose compatibility keys from every value that changes tensor shapes,
+semantics, hooks, or scheduler behavior: task, output shape/count, frames,
+steps/schedule, guidance/CFG, dtype/backend, LoRA/adapter, cache quality policy,
+and model-specific conditions. Reject or schedule separately rather than
+silently using the first request's settings.
+
+## Request-level batching
+
+A request-batched pipeline returns exactly one `DiffusionOutput` per logical
+request. This is model-owned pseudocode: `validate_batch_compatibility()` and
+`_generate_batch()` are placeholders to implement from the official contract,
+not shared APIs to import.
+
+```python
+import torch
+
+from vllm_omni.diffusion.data import DiffusionOutput
+from vllm_omni.diffusion.worker.request_batch import (
+    DiffusionRequestBatch,
+    split_diffusion_output_by_request,
+)
+
+class MyPipeline(torch.nn.Module):
+    supports_request_batch = True
+
+    def forward(self, req: DiffusionRequestBatch) -> list[DiffusionOutput]:
+        validate_batch_compatibility(req.sampling_params_list, req.prompts)
+        common = req.sampling_params_list[0]
+
+        text_prompts: list[str] | None = [
+            prompt
+            if isinstance(prompt, str)
+            else (prompt.get("prompt") or "")
+            for prompt in req.prompts
+        ]
+
+        prompt_fields = DiffusionRequestBatch.collate_prompt_field_map(
+            req.prompts,
+            {"prompt_embeds": None, "prompt_mask": None},
+        )
+        if prompt_fields["prompt_embeds"] is not None:
+            text_prompts = None
+        generators = req.collate_request_generators(
+            common.num_outputs_per_prompt,
+            default_generator=None,
+        )
+        output = self._generate_batch(
+            prompt=text_prompts,
+            prompt_fields=prompt_fields,
+            generators=generators,
+            sampling=common,
+        )
+        return split_diffusion_output_by_request(
+            DiffusionOutput(output=output),
+            req,
+            num_outputs_per_prompt=common.num_outputs_per_prompt,
+        )
+```
+
+The prompt-field collator only handles tensor fields; extract raw text prompts
+and model-specific negative/alias fields separately, as current in-tree batched
+pipelines do. Set a raw prompt field to `None` when the matching precomputed
+embedding is supplied. The exact aliases and generator device depend on the
+model. Use current collators rather than manual `torch.cat` when they cover the
+contract; they reject mixed present/missing tensors and incompatible
+shape/dtype/device.
+
+Before reading `sampling_params_list[0]`, validate every field that must be
+common. Tests must cover:
+
+- two and N requests, independent seeds/generators, one output each;
+- mixed prompt lengths and packed/padded boundaries;
+- multiple outputs per prompt and correct result slicing;
+- compatible versus incompatible shapes, steps, guidance, LoRA, and quality;
+- one request failure without cross-request output or state corruption;
+- output ordering and request IDs under different completion times.
+
+After the model advertises support, candidate serving is:
+
+```bash
+vllm serve '<model>' --omni \
+  --max-num-seqs 4 \
+  --request-batch-max-wait-ms 20
+```
+
+The wait window trades admission latency for batch formation. Benchmark at 0
+and candidate nonzero values under the target arrival process.
+
+## Step execution and continuous batching
+
+### Full protocol skeleton
+
+Use the current shared state types and make every mutable value request-local:
+
+The helpers beginning with `_` and `validate_step_batch()` below are
+model-owned placeholders; scheduler initialization is scheduler-specific.
+
+```python
+import copy
+from collections.abc import Sequence
+from typing import Any, ClassVar
+
+import torch
+
+from vllm_omni.diffusion.data import DiffusionOutput
+from vllm_omni.diffusion.models.interface import SupportsStepExecution
+from vllm_omni.diffusion.worker.input_batch import InputBatch
+from vllm_omni.diffusion.worker.utils import StepRequestState
+
+class MyPipeline(torch.nn.Module, SupportsStepExecution):
+    supports_step_execution: ClassVar[bool] = True
+
+    def prepare_encode(
+        self,
+        state: StepRequestState,
+        **kwargs: Any,
+    ) -> StepRequestState:
+        prepared = self._prepare_request(state.prompt, state.sampling)
+        state.prompt_embeds = prepared.prompt_embeds
+        state.latents = prepared.latents
+        state.timesteps = prepared.timesteps
+        state.scheduler = copy.deepcopy(self.scheduler)
+        state.scheduler.set_begin_index(0)
+        state.step_index = 0
+        state.extra["task_state"] = prepared.task_state
+        return state
+
+    def denoise_step(
+        self,
+        input_batch: InputBatch,
+        *,
+        states: Sequence[StepRequestState] | None = None,
+        **kwargs: Any,
+    ) -> torch.Tensor | None:
+        validate_step_batch(input_batch, states)
+        return self.transformer(**build_batched_denoise_inputs(input_batch))
+
+    def step_scheduler(
+        self,
+        state: StepRequestState,
+        noise_pred: torch.Tensor,
+        **kwargs: Any,
+    ) -> None:
+        timestep = state.current_timestep
+        if timestep is None:
+            raise RuntimeError("step_scheduler called without a current timestep")
+        state.latents = state.scheduler.step(
+            noise_pred,
+            timestep,
+            state.latents,
+            return_dict=False,
+        )[0].to(state.latents.dtype)
+        state.step_index += 1
+
+    def post_decode(
+        self,
+        state: StepRequestState,
+        **kwargs: Any,
+    ) -> DiffusionOutput:
+        return DiffusionOutput(output=self.decode(state.latents))
+```
+
+This is a shape guide, not drop-in code: verify the target revision's
+`StepRequestState.extra`, scheduler signature, CFG handling, state fields, and
+decode contract. Follow a current native pipeline such as Qwen-Image for the
+exact revision.
+
+Never use the shared `self.scheduler` to advance a request. Store scheduler,
+latents, embeddings, timesteps, masks, RNG/generator, task metadata, cache
+policy, adapter, and output assembly state on `StepRequestState` or its
+request-local extension. Pipeline fields may hold immutable weights/config and
+bounded immutable caches only.
+
+### Qualification sequence
+
+1. Compare request-mode and step-mode at `max_num_seqs=1` for every task using
+   identical seed/schedule/shape and intermediate latents.
+2. Abort queued and in-flight requests at several step boundaries.
+3. Increase to two compatible requests; verify independent scheduler/RNG/state
+   and output order.
+4. Increase to the target capacity and mix prompt lengths.
+5. Exercise mixed shape, frames, steps/schedule, task, guidance, and quality.
+   Batch only compatible rows; otherwise split/reject predictably.
+6. Inject a failure in encode, denoise, scheduler, and decode independently.
+7. Run the production soak and fault matrix.
+
+Candidate commands only after capability validation:
+
+```bash
+# Conservative step execution and step-boundary abort.
+vllm serve '<model>' --omni \
+  --step-execution \
+  --max-num-seqs 1
+
+# Experimental batched-step candidate after multi-request E2E.
+vllm serve '<model>' --omni \
+  --step-execution \
+  --max-num-seqs 4
+```
+
+If an attention backend is only correct for single-request step mode, fail
+early or keep the recipe at `max_num_seqs=1`. Do not describe generic runtime
+support as model support.
+
+## Abort and cleanup
+
+### Required semantics
+
+For queued and in-flight abort/disconnect:
+
+- emit exactly one terminal aborted/error result;
+- do not run `post_decode` after abort unless a documented partial-output mode
+  explicitly requires it;
+- do not increment `step_index` or apply a scheduler result after cancellation;
+- remove scheduler/state cache entries and request IDs;
+- release latents, prompt/reference tensors, cache hooks, DLO buffers, connector
+  messages, uploaded/temp files, and output buffers;
+- keep other requests alive;
+- make cleanup idempotent when disconnect, timeout, and worker error race;
+- drop late results/exceptions for already-resolved or cancelled futures
+  without terminating the shared result-pump thread;
+- complete the next valid request with its requested cache/offload policy.
+
+Test cancellation before engine admission, while queued, after encode, during
+early/middle/final denoise steps, before decode, and while returning/persisting
+output. Include async job cancellation and client socket disconnect.
+
+### Failure injection matrix
+
+| Location | Injection | Assertions |
+|---|---|---|
+| Validation/upload | bad MIME, count/bytes overflow, truncated media | 400/413, no job/engine submit, no temp leak |
+| Load/start | missing fused shard, unsupported topology | startup fails, actionable error, no silent init |
+| Encode | exception/OOM/timeout | one error, request state/temp released |
+| Denoise | exception/OOM/worker exit | peers handled, hooks/buffers released, health behavior explicit |
+| Scheduler | invalid state/shape | no latent/step partial commit |
+| Decode/post | exception/codec error | no corrupt successful result, partial output removed |
+| Result/IPC | late result after cancel, large split tensor, orphan SHM | pump remains alive, one terminal result, exact ownership cleanup |
+| Connector/disagg | timeout/retry/duplicate | idempotency, backpressure, no orphan payload |
+
+After each injection, check server health and issue a known-good request. If
+the process must restart, document and test the restart policy; do not claim
+transparent recovery.
+
+### Large-output transport
+
+Treat worker-to-engine IPC and engine-to-client transport as separate hops.
+For batched video, prove that each logical output does not accidentally retain
+or serialize the complete batch storage. Exercise the target revision's shared
+memory or artifact-handle path above and below its threshold, including
+non-contiguous views, multiple outputs, consumer failure, timeout, abort, and
+worker exit. Verify every shared segment/file is released exactly once.
+
+Freeze the media contract at each hop: dtype, value range, shape/layout,
+contiguity, request ownership, encoding/color model, payload bytes, and pending
+consumers. Device-side float-to-uint8 preparation, D2H/IPC, planar/interleaved
+conversion, CPU encode/mux, and delivery off the asyncio event loop are
+different optimizations. Validate their route selection and fallbacks
+independently. Byte-identical HTTP MP4 output does not prove that a raw offline
+tensor kept its dtype or range.
+
+For long-window output, validate segment ordering and audio/video continuity,
+then exercise payloads around serializer, shared-memory, HTTP, and codec size
+limits. Chunking must reassemble exactly, reject malformed/missing chunks, and
+release partial state after abort, timeout, or peer failure.
+
+Qualify chunk production, transport, and public streaming as separate layers.
+A VAE callback must define owned representation, frame offsets, final marker,
+rank ownership, source/config gate, and collective-safe failure, but it does not
+prove SHM/ZMQ delivery, bounded backpressure, cancellation, codec reuse, or
+client streaming. Measure time to first owned chunk, first encoded fragment,
+steady cadence, mux finalization, and complete client artifact independently.
+
+Measure remote MP4 encoding, serialization/copies, network transfer, and client
+materialization separately from denoise/decode. An inference-kernel speedup does
+not establish serving throughput if output transport dominates or leaks.
+
+## Mixed-RPS soak
+
+Use a realistic request mix and arrival distribution. Include small/large
+shapes, short/long prompts, all advertised tasks, cache tiers, and a small rate
+of abort/disconnect/error injections. Run phases below, near, and above measured
+saturation so queue/backpressure behavior is visible.
+
+Example serving benchmark phase:
+
+```bash
+python benchmarks/diffusion/diffusion_benchmark_serving.py \
+  --base-url http://127.0.0.1:8091 \
+  --endpoint /v1/videos \
+  --model '<model>' \
+  --dataset trace \
+  --dataset-path /path/to/versioned-production-trace.jsonl \
+  --task t2v \
+  --num-prompts 500 \
+  --request-rate 0.5 \
+  --max-concurrency 8 \
+  --warmup-requests 8 \
+  --warmup-concurrency 8 \
+  --output-file soak-near-saturation.json
+```
+
+Use a task-appropriate trace/media dataset. Run the same phase at rates selected
+from an initial saturation curve; a literal `0.5` is illustrative only.
+
+On supported GPU hosts, wrap a test or soak with the repository monitor:
+
+```bash
+bash tests/dfx/stability/scripts/resource_monitor.sh run --backend gpu -- \
+  pytest -s -v tests/dfx/stability/scripts/test_stability_<model>.py -m slow
+```
+
+Check the target revision's monitor backends. Do not claim CPU/NPU monitoring
+from a reserved CLI choice unless collection is implemented and verified.
+
+Report:
+
+- offered and achieved RPS, success/error/abort rates;
+- p50/p95/p99 and queue/service/stage times, with the successful sample count
+  and arrival process used for each percentile;
+- throughput per device and batch/wave utilization;
+- per-rank HBM and process-tree PSS trend/slope;
+- cache size/hit rate, temp-file/disk growth, open FDs/handles;
+- outstanding shared-memory/artifact handles and encoded/network bytes;
+- worker/engine health, restarts, OOM/timeouts, backpressure response;
+- total duration, request count/mix, raw logs/JSON/monitor artifact hashes.
+
+A short burst or repeated serial curl loop is not a stability test.
+
+## Four CI tracks
+
+Keep the tracks independent so a failure answers a specific question. These
+four readiness tracks are not the repository's numbered CI levels. Map them
+onto the current L1-L5 taxonomy:
+
+| Readiness track | Repository CI placement |
+|---|---|
+| Function | L1 unit/logic plus L2 basic online/offline E2E; extend real-model scenarios in L3/L4 |
+| Accuracy | L3 real-model gates for important cases; deeper and broader nightly coverage in L4 |
+| Performance | Important thresholds in L3 where they fit the time budget; full baselines/regression jobs in nightly L4 |
+| Reliability | Cheap rejection/recovery assertions can run earlier, but long stability, fault injection, and reliability suites belong to weekly L5 |
+
+Distributed topology is a test case within the appropriate level, not a
+replacement definition for a level.
+
+### 1. Function CI
+
+Cover:
+
+- strict unquantized and advertised quantized loading;
+- fused source-shard completeness and prefix routing;
+- official task positive boundaries and neighboring negative cases;
+- offline, sync, async normalization/parity;
+- single-device reference plus every advertised topology smoke;
+- output type/count/order and request ID isolation;
+- feature rejection paths that must fail early.
+
+### 2. Accuracy CI
+
+Pin official implementation/checkpoint/vLLM-Omni revisions, prompts and media
+hashes, seed, scheduler/sigma/steps, dimensions/frames/FPS, guidance, dtype,
+backend, and topology. Compare at suitable levels:
+
+- component outputs or selected blocks;
+- one or more intermediate denoise latents;
+- final image/video/audio artifact metrics;
+- temporal consistency and audio/video synchronization where relevant.
+
+Set tolerance per modality/hardware/backend with rationale. Keep failure
+artifacts and metadata. A single visual inspection does not define CI.
+
+For quantization trajectory tooling, inspect the target revision's current
+`vllm_omni/quantization/tools/compare_diffusion_trajectory_similarity.py` and
+reuse it when compatible instead of creating an ad hoc comparator.
+
+### 3. Performance CI
+
+Pin workload and best deployment. Include a resident recommended row and a
+small-HBM/DLO row when both are production targets. Declare:
+
+- one explicit warmup exclusion and at least three measured fixed-work
+  repetitions;
+- raw result JSON and environment identity;
+- median/mean plus range for small fixed-work A/Bs; p50/p95/p99 only for an
+  arrival-load run with enough samples, plus throughput/RPS, stage time,
+  HBM/PSS;
+- regression metric, threshold, baseline owner, and update procedure;
+- correctness/accuracy check linked to the same configuration.
+
+Use the current harness config style, for example:
+
+```bash
+pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \
+  --test-config-file tests/dfx/perf/tests/test_<model>_vllm_omni.json
+```
+
+Verify parser options on the target revision. Do not copy a JSON schema from a
+different model without matching its endpoint, task, dataset, and deploy path.
+
+### 4. Reliability CI
+
+Run a bounded CI soak plus a longer scheduled soak. Cover:
+
+- below/near/above-saturation arrival phases;
+- queued/in-flight abort and disconnect;
+- OOM/timeout and worker/process failure appropriate to the platform;
+- DLO/cache/connector cleanup;
+- health, fast-fail/backpressure, restart/recovery policy;
+- memory/temp/FD trend and successful known-good request after each fault.
+
+Keep reliability thresholds independent from latency thresholds. A fast server
+that leaks memory or hangs after cancellation fails production readiness.
+
+## PR evidence checklist
+
+The PR description must include or link:
+
+- official source/checkpoint revisions and API/limit matrix;
+- scoped feature compatibility matrix with the four allowed states;
+- implementation notes for FP8, DLO, request cache, attention/fusion, batching;
+- exact environment, deploy/serve, every task curl, benchmark and soak commands;
+- fixed accuracy inputs and artifacts;
+- raw before/after performance runs and memory traces;
+- output-contract manifests and stage accounting from device preparation
+  through the complete client artifact;
+- abort/disconnect/fault results and cleanup evidence;
+- Function/Accuracy/Performance/Reliability CI files and ownership;
+- limitations/TODOs that remain `not tested` rather than implied support.
+
+Separate the Day-0 milestone from production readiness. If the PR only closes
+some gates, title and description should say which scoped gates it closes.

.claude/skills/readme.md

@@ -26,6 +26,9 @@ self-check.
 
 - [`add-diffusion-model`](add-diffusion-model/SKILL.md): guides integration of
   a new diffusion model into `vllm-omni`
+- [`add-recipe`](add-recipe/SKILL.md): creates or updates in-repository model
+  recipes with verified task/input/output contracts, concrete hardware,
+  shared feature-guide links, exact commands, and qualification evidence
 - [`add-tts-model`](add-tts-model/SKILL.md): covers integration of new TTS
   models and related serving workflows
 - [`diffusion-perf-opt`](diffusion-perf-opt/SKILL.md): guides diffusion model
@@ -37,6 +40,10 @@ self-check.
 - [`precheck-pr`](precheck-pr/SKILL.md): self-checks a branch before creating a
   PR by validating title format, dead code, simplification opportunities,
   accuracy and performance claims, and merge readiness
+- [`production-add-diffusion-model`](production-add-diffusion-model/SKILL.md):
+  takes a working Day-0 diffusion integration through API/limit parity,
+  feature-combination evidence, quantization/offload, performance, production
+  serving, hardware recipes, and CI readiness
 - [`quantization`](quantization/SKILL.md): guides quantization method selection,
   model integration, checkpoint loading, and quality/performance validation
   for vLLM-Omni

.claude/skills/review-pr/SKILL.md

@@ -123,10 +123,11 @@ it changes again, report the churn and wait for a stable target.
 
 For a trusted PR head, materialize the pinned head in an isolated detached
 worktree. A worktree freezes identity but is not a security sandbox. Treat fork
-heads as untrusted unless the user and environment policy explicitly establish
-otherwise: execute them only in a disposable, secret-free sandbox with restricted
-filesystem, network, and resources; without one, use static SHA-addressed reads
-and CI evidence only. For a local review, freeze the committed, index, worktree,
+heads as untrusted until the user and environment policy explicitly establish
+trust: execute their code only after that trust is recorded for this host,
+with the trusted SHA noted in the review state and secrets kept out of the
+execution scope; without recorded trust, use static SHA-addressed reads and CI
+evidence only. For a local review, freeze the committed, index, worktree,
 and NUL-safe in-scope untracked contents. Follow
 [review-execution.md](references/process/review-execution.md) for trust gates,
 state fingerprints, and byte-for-byte staleness checks.
@@ -196,7 +197,7 @@ style preferences as simplification findings.
 
 Before each validation group, verify the frozen SHA plus the tracked, index,
 untracked, and ignored-file fingerprint, or recreate a pristine snapshot. On a
-trusted head or inside the required sandbox, run an import/version preflight,
+head the user explicitly trusted for this host, run an import/version preflight,
 then the narrowest relevant tests and low-cost static checks. Bind every result
 to the head SHA, snapshot fingerprint, and environment fingerprint. Never run
 imports, tests, builds, hooks, or repo-configurable tooling from an untrusted

.claude/skills/review-pr/references/checks/perf-verification.md

@@ -22,6 +22,19 @@ Keep exact commands and report variability rather than a cherry-picked run.
 Use the repository's benchmark when it covers the claim; otherwise use the
 smallest reproducible workload that reaches the changed production path.
 
+## Isolation per claim
+
+Isolate one claim per comparison. When a PR stacks several changes (a fused or
+restructured load path, a distilled step schedule, a new cache), request one
+column per claim at equal steps/workload instead of one blended base-vs-head
+delta, and name the unchanged stage or metric that pins equivalence.
+
+When the changed path is runtime-switchable (feature flag or environment
+variable), run every column on the same head and toggle the switch rather than
+diffing against a base commit that cannot separate the stacked changes:
+base, head with the switch off, head with the switch on, then head with the
+switch on plus each further claim.
+
 ## Evidence table
 
 | Dimension | Typical evidence |
@@ -34,6 +47,100 @@ smallest reproducible workload that reaches the changed production path.
 Every optimization needs correctness or quality evidence. A faster result on a
 different workload, precision, seed, or topology is not a valid comparison.
 
+## E2e stage-attribution table
+
+For an end-to-end latency claim, request the stage-attribution table the
+repository's perf PRs use:
+
+- Rows carry the benchmark's own field names: total e2e, per-stage execution,
+  postprocess, response encoding, and a request-to-output roll-up. Add a stage
+  handoff/idle row when the pipeline spans stages — connector transfer and
+  scheduling gaps are invisible in per-stage latency deltas.
+- Sanity rows pin comparability: inference steps, output frames/pixels/samples,
+  and generated tokens held identical across columns.
+- Report P50 and P100 from the stated warmup and measured counts. One
+  population per number: never mix cold-start and warm samples into one value.
+- Name the baseline and candidate commit SHAs, and close with an isolation
+  statement naming the metric that stayed unchanged ("X remained effectively
+  unchanged, isolating the gain to Y").
+
+When the serving path caches, captures, or regenerates at runtime (CUDA-graph
+bucket growth, cache rebuilds, recompilation), medians hide the resulting tail.
+Also request the event count and the maximum observed stall.
+
+Component-level tables pair with, not replace, the e2e table: per-op reference
+vs candidate timings (with the numerical-equality result per op) plus one
+complete-pipeline check under the production shapes.
+
+## Proportionality
+
+When a PR adds a model and explicitly labels its numbers as smoke observations
+rather than claims, do not demand the attribution machinery. Ask only for
+evidence protecting the model's stated operating contract — for a
+realtime/duplex model that is sustained cadence over a realistic session
+(enough ticks to leave warmup), with any mid-session capture/regeneration
+event and its stall reported, since a cadence that collapses mid-session is a
+functional failure, not a perf claim.
+
+## Realtime/duplex world models
+
+Request the operating-contract evidence as serving metrics, not claims:
+
+- Sustained per-tick latency over one realistic session: median, max, and the
+  argmax tick.
+- Stall events with their causes.
+- Peak memory at first versus last tick — session-owned state must not grow
+  per tick.
+- Control latency from a submitted action to the tick that applies it.
+- Realtime-versus-offline parity under the same controls, using the
+  repository's video-similarity helpers. Parity plotted over tick index is the
+  drift-over-horizon curve and the honest measure of session lifetime;
+  distribution-level suites (FVD, VBench) belong to model evaluation, not the
+  serving integration.
+
+Watch the contract boundaries the model documents: if a control input is
+tick-API-only, the parity pair must use a horizon both paths can run.
+
+## Speech-generation (TTS) models
+
+The streaming contract outranks the average — a synthesis that pauses
+mid-stream is broken for realtime use regardless of its averages. Request:
+
+- The benchmark's own speech metrics over one long utterance (enough audio to
+  leave warmup, tens of seconds): `audio_ttfp` (time to first packet),
+  `audio_rtf` (real-time factor; above 1.0 cannot keep up), `e2el`, and
+  `ttft`/`tpot` where tokens precede audio.
+- Mid-utterance stalls as the benchmark's continuity fields, not prose:
+  `audio_underrun` percentiles and the `audio_continuity_ok` rate
+  (`vllm_omni/benchmarks/metrics/metrics.py`), computed by
+  `vllm_omni/benchmarks/audio_continuity.py` — the companion to `audio_ttfp`
+  and `audio_rtf`. A synthesis that bursts chunks fast enough on average but
+  underruns mid-stream fails the streaming contract even when the averages
+  pass.
+- Quality as a paired comparison against the reference implementation under
+  the same text, speaker prompt, and seed; report sample rate and frame
+  count, and assert output completeness — truncated or dropped trailing audio
+  is a known failure class, not a quality nuance.
+- For realtime or duplex TTS: interruption (barge-in) latency and the
+  per-session memory trend across turns.
+
+## Omni models
+
+Apply the e2e stage-attribution table directly to the voice-to-voice path of
+omni (speech-to-speech and multimodal AR) models:
+
+- TTFT for text output, `audio_ttfp` for audio output, and per-stage
+  ITL/TPOT cadence across the perception-understanding-generation chain.
+- Cross-modal parity — the omni analogue of realtime-versus-offline parity:
+  the same semantic request through text and audio inputs, compared on output
+  content or timing.
+- For duplex omni sessions: turn-taking latency, interruption handling, the
+  concurrent session count actually validated, and a per-turn memory trend —
+  session-owned KV state must not grow per turn. When either turn produces
+  audio, continuity is judged with the same `audio_underrun` and
+  `audio_continuity_ok` fields as TTS, since turn-taking latency alone does
+  not expose mid-utterance underruns.
+
 ## Reviewer verification ladder
 
 1. Run base/head A/B on suitable hardware when affordable.

.claude/skills/review-pr/references/process/general-checks.md

@@ -121,6 +121,19 @@ Search bounded callers and sibling implementations before reporting dead code,
 duplication, or a missed consumer. Keep new abstractions only when they have a
 distinct live caller, invariant, or compatibility need.
 
+Verify absence and equality claims with whole-file evidence. A line window, a
+grep for one symbol, or a commit message proves presence, never absence: to
+assert that a file matches another version or lacks a change, diff or hash the
+full files (for example `git diff <sha1>:<path> <sha2>:<path>` or the contents
+API's size and SHA fields) before anchoring the finding. A verification
+shortcut that would itself be wrong to run is not evidence.
+
+For a value written or finalized in more than one place, establish which site
+executes first on each entry path — and whether the earlier site makes the
+later ones unreachable — before naming where a fix belongs. Enumerating all
+call sites correctly still misses the bug when a path the enumeration ranked
+last actually finalizes first and clears the state the proposed fix checks.
+
 ## Finding bar
 
 Anchor each finding to a changed `path:line`; name the trigger or call path,

.claude/skills/review-pr/references/process/review-execution.md

@@ -39,13 +39,13 @@ isolated detached worktree. Run every source read, `rg` search, import, and test
 from that snapshot, not the caller's checkout. A detached worktree provides
 snapshot isolation, not a security boundary.
 
-Treat a fork head as untrusted unless the user and environment policy explicitly
-establish trust. Never run its imports, tests, builds, hooks, package setup, or
-repo-configurable linters/plugins on the reviewer host. Execute them only in a
-disposable sandbox or VM with no credentials or inherited secrets, no host agent
-or service sockets, minimal read-only host mounts, disabled or explicitly
-allowlisted network, resource/time limits, and destruction after the run. If
-that boundary is unavailable, limit the review to the remote diff, SHA-addressed
+Treat a fork head as untrusted until the user and environment policy
+explicitly establish trust. Without recorded trust, never run its imports,
+tests, builds, hooks, package setup, or repo-configurable linters/plugins on
+the reviewer host. Once the user explicitly trusts this pinned commit for
+this host, record the trusted SHA with the review state and execute with
+credentials, agent sockets, and other secrets out of scope. Without recorded
+trust, limit the review to the remote diff, SHA-addressed
 reads such as `git show <head_sha>:<path>`, and existing CI evidence; report all
 executable validation as a gap.
 

.gitignore

@@ -281,3 +281,20 @@ lance_compare/
 
 # Linting binaries downloaded by local pre-commit hooks
 shellcheck*/
+
+# --- edge-infer -------------------------------------------------------------
+# Measurement artifacts are the substance of this fork, not build output: every
+# published figure is traceable to a file below. They are tracked despite the
+# blanket `*.json` and `results/` rules above. Binary captures (.npz, .onnx,
+# .wav, .npy) stay ignored.
+!benchmarks/edge_harness/results/
+!benchmarks/edge_harness/results/**
+!baselines/
+!baselines/**
+benchmarks/edge_harness/results/**/*.npz
+benchmarks/edge_harness/results/**/*.onnx
+benchmarks/edge_harness/results/**/*.npy
+benchmarks/edge_harness/results/**/*.wav
+
+# Portable wire fixtures are source inputs, not generated measurements.
+!packages/omni-stage-contracts/conformance/*.json

CONTRIBUTING.md

@@ -2,6 +2,8 @@
 
 You may find information about contributing to vLLM-Omni on [Contributing](https://vllm-omni.readthedocs.io/en/latest/contributing/).
 
+AI-assisted contributions must follow the [AI Usage Policy](docs/contributing/README.md#ai-usage-policy), including disclosure, human review, and validation requirements.
+
 Local `pre-commit` gates (SPDX, forbidden imports, `torch.cuda`, shellcheck,
 mypy, test marks, TTS adapter ratchet, markdownlint, Buildkite schema, and
 how to extend allowlists) are documented in the

NOTICE

@@ -0,0 +1,34 @@
+edge-infer
+Copyright 2026 tzhouam
+
+This product is a derivative work of vLLM-Omni
+(https://github.com/vllm-project/vllm-omni), licensed under the Apache License,
+Version 2.0. Upstream code is Copyright the vLLM-Omni contributors, and vLLM-Omni
+is itself a derivative of vLLM (https://github.com/vllm-project/vllm).
+
+Modifications in this fork are confined to edge/CPU inference and are listed in
+README.md; the substantive ones are:
+
+  vllm_omni/model_executor/layers/quantization/cpu_int4.py
+      4-bit weight-only quantization for the CPU backend
+  vllm_omni/model_executor/models/spark2_5/spark2_5.py
+      RoPE table sized to max_model_len rather than the advertised context
+  vllm_omni/edge/
+      device-class weight-path selection, KV budget sizing, memory release
+  vllm_omni/platforms/cpu/, vllm_omni/patch.py
+      CPU platform and CPU-backend patches
+  benchmarks/edge_harness/
+      measurement tooling and results (original to this fork)
+  baselines/, docs/engines/, docs/analysis/
+      benchmark results and written analyses (original to this fork)
+
+vLLM itself is not modified and no vLLM source is redistributed here. This fork
+changes vLLM's CPU behaviour through its own extension points at runtime
+(vllm_omni/patch.py); the pinned upstream versions are recorded in
+docs/edge/environment.md.
+
+Baseline measurements reference llama.cpp (https://github.com/ggml-org/llama.cpp,
+MIT). No llama.cpp source is redistributed; only benchmark output produced with
+it, under baselines/llama_cpp/.
+
+Upstream's README is preserved at docs/README.vllm-omni.md.

README.md

@@ -1,107 +1,116 @@
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/logos/vllm-omni-logo.png">
-    <img alt="vllm-omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/logos/vllm-omni-logo.png" width=55%>
-  </picture>
-</p>
-<h3 align="center">
-Easy, fast, and cheap omni-modality model serving for everyone
-</h3>
+# edge-infer
 
-<p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://deepwiki.com/vllm-project/vllm-omni"><b>DeepWiki</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/1aPj0OGl_-ZVoib-Qne5dGDAlrRFB-PdHl6E-EE99g8E/edit?usp=sharing"><b>Slides</b></a> |
-</p>
+Running multimodal models on **edge CPUs** — a mini-PC or a gateway box with
+tens of gigabytes of RAM and no GPU — instead of on servers.
 
----
+Built as a fork of [vLLM-Omni](https://github.com/vllm-project/vllm-omni),
+because it is the only engine with a multi-stage multimodal pipeline,
+continuous batching and an OpenAI-compatible server; the alternatives were
+evaluated and the reasoning is in [docs/engines/](docs/engines/).
 
-*Latest News* 🔥
+The work turned out to be less about kernels than about three things nobody
+profiles: how much memory is resident while the model *loads*, how much is
+resident that nothing ever reads, and whether the number you just measured came
+from the code path you thought it did. The largest single win here — 843 MB —
+was a setting that had been marked "no effect, do not retry" because the code
+path under test discarded it.
 
-- [2026/08] [VeRL-Omni](https://github.com/verl-project/verl-omni) `v0.2.0` is released: faster diffusion RL powered by vLLM-Omni (request-level/step-wise batching with FA3), rebuilt Qwen3-Omni multimodal training (DPO & GSPO), plus LTX-2.3, Qwen-Image-Edit support and more. See the [release notes](https://github.com/verl-project/verl-omni/releases/tag/v0.2.0).
-- [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support.
-- [2026/07] We released [0.24.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.24.0) - aligned with the vLLM 0.24 release line, expanding production-ready coverage across TTS, speech, diffusion, image/video generation, and robot-policy serving, with major Omni stage runtime refactoring, diffusion request-level batching, async output materialization, quantization/cache/memory improvements, and broad CUDA/ROCm/XPU/NPU support.
-- [2026/06] Starting with [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0), vLLM-Omni publishes a stable release aligned with every even-numbered upstream vLLM minor version. [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0), [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0), [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0), and [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) continued this cadence, expanding omni and world-model support with [NVIDIA Cosmos3](recipes/cosmos3/Cosmos3-Nano.md) and DreamZero, adding models such as MiniCPM-o 4.5, MOSS-TTS, and Lance, and advancing TTS, diffusion, distributed execution, quantization, RL integration through [VeRL-Omni](https://github.com/verl-project/verl-omni), and CUDA/ROCm/MUSA/NPU/XPU coverage.
-- [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
-- [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.
+This repository holds the engine changes, the measurement harness, the raw
+results, the baselines they are compared against, and the written analyses.
+Upstream's own README is preserved at [docs/README.vllm-omni.md](docs/README.vllm-omni.md).
 
----
+## What's here
 
-## About
+| area | where |
+|---|---|
+| 4-bit CPU quantization | `vllm_omni/model_executor/layers/quantization/cpu_int4.py` — W4A16 on PyTorch's tinygemm kernel, prefill reading the kernel's own weight layout instead of a duplicate |
+| Memory | RoPE tables sized to the servable context; weights repacked in blocks; a KV budget that is actually honoured on CPU |
+| CPU platform + vLLM patches | `vllm_omni/platforms/cpu/`, `vllm_omni/patch.py` — vLLM is monkey-patched at import; its *source* is unmodified and no vLLM is redistributed. See [docs/edge/environment.md](docs/edge/environment.md) |
+| Device routing | `vllm_omni/edge/weight_path.py` picks the weight format per device class, from measurements rather than assumption |
+| Device placement | `vllm_omni/edge/placement.py` decides which parameters leave the GPU when the checkpoint is bigger than the VRAM — cold ones first, because `embed_tokens` and `lm_head` are the same size and the opposite decision |
+| Measurement harness | `benchmarks/edge_harness/` — the grid runner, benchmarks, memory ledger, fidelity metric and agent suite every number here came from |
+| Raw results | `benchmarks/edge_harness/results/` — 100+ artifacts backing the published figures, each with provenance |
+| Baselines | `baselines/` — llama.cpp runs and the comparison scripts |
+| Engine analyses | `docs/engines/` — llama.cpp, ExecuTorch, MNN, ncnn, vLLM, vLLM-Omni |
+| Study documents | `docs/analysis/` — the full reports these summaries distil |
 
-[vLLM](https://github.com/vllm-project/vllm) was originally designed to support large language models for text-based autoregressive generation tasks. vLLM-Omni is a framework that extends its support for omni-modality model inference and serving:
+## Headline results
 
-- **Omni-modality**: Text, image, audio, video, and action data processing
-- **Non-autoregressive Architectures**: extend the AR support of vLLM to Diffusion Transformers (DiT) and other parallel generation models
-- **Heterogeneous outputs**: from traditional text generation to multimodal and action outputs
+Spark-X2.5-1.7B, Xeon Platinum 8480C, 24 pinned node-local cores, W4A16 g128.
+Every figure is a median of at least three interleaved passes against a control
+arm, on a shared host where end-to-end throughput swings ~30% run to run.
 
-<p align="center">
-  <picture>
-    <img alt="vllm-omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/omni-modality-model-architecture.png" width=55%>
-  </picture>
-</p>
+**Memory** — each measured as its own A/B with the alternative reachable by a flag:
 
-vLLM-Omni is fast with:
+| lever | saving | cost |
+|---|---|---|
+| KV budget 1 GiB → 192 MiB | **843 MB** | fewer cached tokens |
+| RoPE table capped to `max_model_len` | **618 MB** | none |
+| One packed weight layout instead of two | **615 MB** | ~1.6× prefill |
+| Blocked weight repack | **489 MB** of load-time peak | none |
 
-- State-of-the-art AR support by leveraging efficient KV cache management from vLLM
-- Pipelined stage execution overlapping for high throughput performance
-- Fully disaggregation based on OmniConnector and dynamic resource allocation across stages
+These overlap and **must not be added** — see [docs/edge/results.md](docs/edge/results.md).
 
-vLLM-Omni is flexible and easy to use with:
+**Speed**, same machine:
 
-- Heterogeneous pipeline abstraction to manage complex model workflows
-- Seamless integration with popular Hugging Face models
-- Tensor, pipeline, data and expert parallelism support for distributed inference
-- Streaming outputs
-- OpenAI-compatible API server
-- Full-duplex realtime serving with streaming audio input and output (experimental)
+| path | decode | prefill (pp512) |
+|---|---|---|
+| W4A8 (AMX `int4_scaled_mm_cpu`) | 74.4 tok/s | — |
+| W4A16 (tinygemm) | 71.6 tok/s | 2190 tok/s |
+| `cpu_gemm_wna16` fallback | 32.3 tok/s | — |
+| llama.cpp Q4_K_M (reference) | 90.0 tok/s | 1121 tok/s |
 
-vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
+Decode is llama.cpp's; prefill is ours by ~2×. Neither engine is
+bandwidth-bound — both sit at 33–39% of the measured 222 GB/s ceiling — so the
+decode gap is the execution model, not the kernels. See
+[docs/edge/comparisons.md](docs/edge/comparisons.md).
 
-- **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
-- **TTS models** (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
-- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
-- **Robot-policy and action models** (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)
+## Why the numbers here are worth reading
 
-## Getting Started
+This project has retracted five conclusions, every one of them a measurement
+error rather than a coding error. The harness exists because of them:
 
-Visit our [documentation](https://vllm-omni.readthedocs.io/en/latest/) to learn more.
+- a "3.1 GB saving" that was `VmRSS` counting the mmap'd checkpoint;
+- W4A8 timings taken on a checkpoint whose output head was silently left in bf16;
+- a memory benchmark reading `/proc/self/status` while the model ran in a *worker process*;
+- "KV sizing saves only 188 MB" — measured through a code path that discarded the setting, so both arms ran the same cache;
+- a 1.9× prefill cost quoted from the one pass of a grid whose other passes had crashed.
 
-- [Installation](https://vllm-omni.readthedocs.io/en/latest/getting_started/installation/)
-- [Quickstart](https://vllm-omni.readthedocs.io/en/latest/getting_started/quickstart/)
-- [List of Supported Models](https://vllm-omni.readthedocs.io/en/latest/models/supported_models/)
-- [Deployment Recipes](https://recipes.vllm.ai) for vLLM-Omni model serving
+[docs/edge/methodology.md](docs/edge/methodology.md) is the resulting discipline,
+and it is the most transferable thing in this repository.
 
-## Contributing
+## Quick start
 
-We welcome and value any contributions and collaborations.
-Please check out [Contributing to vLLM-Omni](https://vllm-omni.readthedocs.io/en/latest/contributing/) for how to get involved.
+```bash
+# CPU build of vLLM, then this fork
+pip install vllm==0.28.0+cpu
+VLLM_TARGET_DEVICE=cpu pip install -e .
 
-## Citation
-
-If you use vLLM-Omni for your research, please cite our [paper](https://arxiv.org/abs/2602.02204):
-
-```bibtex
-@article{yin2026vllmomni,
-  title={vLLM-Omni: Fully Disaggregated Serving for Any-to-Any Multimodal Models},
-  author={Peiqi Yin, Jiangyun Zhu, Han Gao, Chenguang Zheng, Yongxiang Huang, Taichang Zhou, Ruirui Yang, Weizhi Liu, Weiqing Chen, Canlin Guo, Didan Deng, Zifeng Mo, Cong Wang, James Cheng, Roger Wang, Hongsheng Liu},
-  journal={arXiv preprint arXiv:2602.02204},
-  year={2026}
-}
+# a right-sized edge deployment
+VLLM_CPU_KVCACHE_SPACE=1 python -c "
+from vllm import LLM, SamplingParams
+llm = LLM(model='<your-cpu_int4-checkpoint>', quantization='cpu_int4',
+          dtype='bfloat16', max_model_len=2048, max_num_seqs=1,
+          kv_cache_memory_bytes=192 << 20)   # the single largest memory lever
+print(llm.generate(['Hello'], SamplingParams(max_tokens=32))[0].outputs[0].text)
+"
 ```
 
-## Join the Community
+## Documentation
 
-Feel free to ask questions, provide feedbacks and discuss with fellow users of vLLM-Omni in `#sig-omni` slack channel at [slack.vllm.ai](https://slack.vllm.ai) or vLLM user forum at [discuss.vllm.ai](https://discuss.vllm.ai).
+| | |
+|---|---|
+| [docs/edge/results.md](docs/edge/results.md) | every measured figure, with spread and baseline |
+| [docs/edge/methodology.md](docs/edge/methodology.md) | how to measure on a shared machine without fooling yourself — the five retractions and the rules they left |
+| [docs/edge/cpu-4bit.md](docs/edge/cpu-4bit.md) | the two 4-bit paths, the packed layout as measured, building a checkpoint |
+| [docs/edge/comparisons.md](docs/edge/comparisons.md) | against llama.cpp and the other engines considered |
+| [docs/edge/environment.md](docs/edge/environment.md) | pins, machine, and why there is no vLLM patch series |
+| [docs/edge/local-text-mode.md](docs/edge/local-text-mode.md) | the single-stage local text engine: device capabilities, memory admission, explicit refusals, cancellation |
+| [docs/edge/external-stage-backends.md](docs/edge/external-stage-backends.md) | placing a stage on the Radeon 890M or the AMD NPU: out-of-process workers, proven placement, the shared-RAM ledger |
+| [docs/edge/harness.md](docs/edge/harness.md) | running a comparison |
+| [docs/edge/laptop-heterogeneous.md](docs/edge/laptop-heterogeneous.md) | Qwen3.8-27B and Spark-X2.5-4B on a Ryzen AI laptop — what the CPU, NPU and GPU are each worth when decode is bandwidth-bound |
+| [ROADMAP.md](ROADMAP.md) | what is next, with expected gains and how each would be verified — and what was rejected |
 
-## Star History
+## Licence
 
-<a href="https://www.star-history.com/?repos=vllm-project%2Fvllm-omni&type=date&legend=top-left">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=vllm-project/vllm-omni&type=date&theme=dark&legend=top-left&sealed_token=ExgLDZJoQEg27Zfhhut2LqN0GYO6Fw2PWLwPE6JYBUp2BgM3hmsYlwaIVopnUEfbRXidQ4nisumrTdKYydiKhy1SZXipw47qY2_tiUDhCpsPXeXtPuEVKVzBwKs3pw0tiHsJgtSfwXx5yjHXck0Y2SblzFWeJYCkTe1WLGTbUAOIETjXXQJjyCGZvKz5" />
-    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=vllm-project/vllm-omni&type=date&legend=top-left&sealed_token=ExgLDZJoQEg27Zfhhut2LqN0GYO6Fw2PWLwPE6JYBUp2BgM3hmsYlwaIVopnUEfbRXidQ4nisumrTdKYydiKhy1SZXipw47qY2_tiUDhCpsPXeXtPuEVKVzBwKs3pw0tiHsJgtSfwXx5yjHXck0Y2SblzFWeJYCkTe1WLGTbUAOIETjXXQJjyCGZvKz5" />
-    <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=vllm-project/vllm-omni&type=date&legend=top-left&sealed_token=ExgLDZJoQEg27Zfhhut2LqN0GYO6Fw2PWLwPE6JYBUp2BgM3hmsYlwaIVopnUEfbRXidQ4nisumrTdKYydiKhy1SZXipw47qY2_tiUDhCpsPXeXtPuEVKVzBwKs3pw0tiHsJgtSfwXx5yjHXck0Y2SblzFWeJYCkTe1WLGTbUAOIETjXXQJjyCGZvKz5" />
-  </picture>
-</a>
-
-## License
-
-Apache License 2.0, as found in the [LICENSE](./LICENSE) file.
+Apache-2.0, inherited from vLLM-Omni. See [LICENSE](LICENSE) and [NOTICE](NOTICE).

ROADMAP.md

@@ -0,0 +1,107 @@
+# Roadmap
+
+Ordered by whether the verification method is *correct*, not by expected size.
+That ordering is deliberate: this project retracted five conclusions, and in
+every case the fix was to the measurement rather than the code.
+
+Each item states the measurement it starts from, what it should gain, and how
+that gain would be verified. An item without a verification method is not on
+this list.
+
+## Done
+
+| | item | result |
+|---|---|---|
+| H1 | Task oracles judged by device state, with variants | alarm oracle needed a *saved* alarm, not a typed `7`; a Wi-Fi round trip now fails the already-satisfied task |
+| H2 | Harness semantics | arm order really rotates; `best` respects metric direction; provenance stamped; failures recorded |
+| H3 | The three missing measurements | `ready_s`, peak `RssAnon`, memory at a filled context — all previously unmeasurable |
+| H4 | Contention guard | aggregate stolen capacity + straggler rule, on the target cores rather than host load average |
+| H5 | Regression gates | bands from observed spread; throughput explicitly ungateable at ~30% swing |
+| H6 | Fidelity metric that ranks | top-1 agreement and KL, replacing a saturating 12-prompt greedy check |
+| H8 | Retracted numbers purged from source | two modules were still shipping withdrawn figures as documentation |
+| I0 | Blocked weight repack | **489 MB** of load-time peak, no cost |
+| I1 | One packed weight layout | **615 MB** resident, at ~1.63× prefill |
+| I2 | Measure `cpu_gemm_wna16` | 2.2× slower than tinygemm; routing was right, now for a measured reason |
+| I7 | Memory ledger, by attribution | found a **618 MB** RoPE table and an **843 MB** KV cache, both inside an "unexplained 1.6 GB" |
+
+## Next
+
+**I6 — import cost.** `import vllm_omni` is ~30 s against `import vllm`'s 5.2 s,
+with transformers' auto image-processing (4.6 s) and the `openai` client (1.3 s)
+identifiable inside it, for a text model that needs neither. Every stage
+subprocess pays it again.
+*Expect:* an upper bound of ~6 s per process; deferred imports often move cost
+rather than remove it. *Verify:* `ready_s`, which now exists — the old timer
+started after imports, so the benchmark meant to prove this could not see it.
+
+**Drop the row-major embedding copy.** `embed_tokens.weight_packed` is 128 MB,
+kept solely because the tinygemm layout was believed un-addressable for a
+single-row lookup. That premise is now known false. *Verify:* `RssAnon` plus an
+embedding-output SNR assertion.
+
+**I10 first step — why is the GEMM at 60%?** The decode GEMM sustains 132 GB/s
+against a measured 222 GB/s ceiling. Part is arithmetic (tinygemm dequantizes
+per activation row) and part may be that each of 113 calls streams only a few MB,
+too little for the prefetcher to reach steady state. Attribute before optimising.
+*Expect:* unknown — a diagnosis, whose output is whether the 2.76 ms implied by
+the gap is reachable at all.
+
+**I3 — asymmetric W4A8.** The symmetric grid is our exporter's choice, not the
+kernel's limit; `models/spark2_5-gptq-asym-g128` already exists. The cheapest
+untried fidelity experiment, and it tests the *stated* cause of W4A8's error.
+*Verify:* `fidelity_kl.py` and the scored agent suite.
+
+**I5 — attention wrapper consolidation.** Four ops per layer, 1.31 ms/step, of
+which 0.83 ms is wrapper self time; at batch 1 the cache-update loop runs two
+iterations while the wrapper rebuilds context and allocates a dummy tensor for
+compile ordering. *Expect:* 0.3–0.5 ms, revised down from an earlier 0.6–0.9 ms
+that assumed a workload never profiled. *Verify:* per-op profile **and**
+unprofiled step latency — one outer call that still launches two threaded
+kernels removes neither thread launch.
+
+**I4 — a better quantizer.** Ours reproduces 2/12 greedy prompts against its own
+bf16; Q4_K_M manages 4/12 — but at 4.5 bits against our 4.25, so close the
+budget gap first (g64, or fp16 scale+zero, both config changes) before
+implementing error-compensated rounding. Timebox it; a null result is worth
+recording.
+
+## Considered and rejected
+
+**A CPU megakernel.** On GPU this fuses a forward pass into one launch to remove
+launch overhead and the cross-kernel prefetch bubble. On CPU the analogues are
+OpenMP fork/join and ATen dispatch — and llama.cpp already is one:
+`ggml_threadpool` starts threads once and they walk the whole graph,
+synchronising at `ggml_barrier` through an atomic spin, where vLLM enters a
+separate `omp parallel` region per kernel.
+
+Measured before deciding: a parallel region costs ~1.2 µs at 24 threads, and
+spin-waiting does not help (`OMP_WAIT_POLICY=ACTIVE` moved it 1.23 → 1.18 µs), so
+it is barrier cost rather than thread wake-up. At a few hundred regions per
+decode step that is 0.3–0.4 ms, 2–3% — the same money as I5, far more cheaply.
+The larger prize is the streaming bubble, but capturing it means replacing the
+execution model inside vLLM's CPU backend, which reimplements ggml. Not next.
+
+**More kernel fusion.** Four attempts, four losses: vLLM's fused CPU RMSNorm is
+3.6–4.5× faster in isolation and slower in situ (inductor already fuses the
+native form with the residual add); fused `gelu_and_mul` 2.7× slower at Spark's
+shape; `cpp.min_chunk_size` −6% decode; an inline gate reduction −44%.
+
+**MiniCPM-o CPU overlay.** The 8 B thinker at bf16 is ~19 GB, so it buys no edge
+deployability. The argument for it — that three-stage parallel init should beat
+the two-stage 2.1× — is also not how overlap works: the bound is ΣT/max(T).
+
+**Speculative decoding for tg128.** It would inflate a fixed-prompt `ignore_eos`
+benchmark without representing anything. Measure it on the agent suite or not at
+all.
+
+**Thread count and warm-up batch size as memory levers.** 64 MB and 2 MB.
+
+## Harness work still open
+
+- **A real device in the agent loop.** The toolkit has 38 tests and has never
+  touched a phone — this host has no `adb`, no emulator, no `/dev/kvm`. The
+  real-device path and per-task resets exist; what is untested is whether a real
+  Settings app matches the fixtures.
+- **Success rates, not single runs.** The build comparison that produced
+  "navigation survives quantization, completion does not" is one run per cell.
+  Variants now exist so repeats are trials; they need running.

baselines/README.md

@@ -0,0 +1,32 @@
+# Baselines
+
+Measurements of other engines and of unquantized references, kept so the
+comparisons in [docs/edge/comparisons.md](../docs/edge/comparisons.md) can be
+checked rather than taken on trust.
+
+## `llama_cpp/`
+
+llama.cpp is the reference for CPU inference of a quantized model. Pinned
+commit and build flags are in
+[docs/edge/environment.md](../docs/edge/environment.md); no llama.cpp source is
+redistributed here, only output produced with it.
+
+| file | what |
+|---|---|
+| `llamacpp_q4km_t{4,8,16,32}.json` | Q4_K_M across thread counts |
+| `llamacpp_grid_t*.json`, `llamacpp_node1_t*.json` | thread and NUMA-node sweeps |
+| `ab_llamacpp_*.json` | interleaved A/B against vLLM on the same cores |
+| `il_llamacpp_p{1,2,3}.json` | the interleaved passes behind the headline decode figure |
+| `cmp_llamacpp.py` | the comparison driver |
+
+Both engines are pinned with `taskset` to the same physical cores and run
+strictly one at a time; vLLM's prefix caching is disabled, because `llama-bench`
+has no such cache and leaving it on would not be a comparison.
+
+## Quality references
+
+`quality_*.json` and `quality_cmp_*.json` hold the greedy-divergence comparison:
+each build against its **own** bf16 output over 12 prompts, plus the
+cross-engine roll-up. Read the caveats in
+[docs/edge/comparisons.md](../docs/edge/comparisons.md) before quoting these —
+the bit budgets are not matched, and the metric saturates.

baselines/llama_cpp/ab_llamacpp_after.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T18:59:20Z",
+    "avg_ns": 580079289,
+    "stddev_ns": 22445145,
+    "avg_ts": 883.695051,
+    "stddev_ts": 34.173372,
+    "samples_ns": [ 550140411, 579685176, 611186497, 588372185, 571012179 ],
+    "samples_ts": [ 930.671, 883.238, 837.715, 870.197, 896.653 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T18:59:24Z",
+    "avg_ns": 2118395683,
+    "stddev_ns": 31881177,
+    "avg_ts": 60.434035,
+    "stddev_ts": 0.909662,
+    "samples_ns": [ 2110731151, 2152277901, 2078510078, 2149464130, 2100995155 ],
+    "samples_ts": [ 60.6425, 59.4719, 61.5826, 59.5497, 60.9235 ]
+  }
+]

baselines/llama_cpp/ab_llamacpp_before.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T18:50:11Z",
+    "avg_ns": 453009837,
+    "stddev_ns": 4617262,
+    "avg_ts": 1130.311918,
+    "stddev_ts": 11.483962,
+    "samples_ns": [ 459248279, 451575469, 456242077, 449875010, 448108352 ],
+    "samples_ts": [ 1114.87, 1133.81, 1122.21, 1138.09, 1142.58 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T18:50:14Z",
+    "avg_ns": 1475847924,
+    "stddev_ns": 9833345,
+    "avg_ts": 86.732884,
+    "stddev_ts": 0.578067,
+    "samples_ns": [ 1489621433, 1475111972, 1476264136, 1476396077, 1461846002 ],
+    "samples_ts": [ 85.9279, 86.7731, 86.7054, 86.6976, 87.5605 ]
+  }
+]

baselines/llama_cpp/ab_llamacpp_t24.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T18:44:52Z",
+    "avg_ns": 487743656,
+    "stddev_ns": 25108865,
+    "avg_ts": 1051.969118,
+    "stddev_ts": 54.409083,
+    "samples_ns": [ 484545979, 492513415, 522973237, 486232711, 452452942 ],
+    "samples_ts": [ 1056.66, 1039.57, 979.018, 1052.99, 1131.61 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T18:44:55Z",
+    "avg_ns": 1589326576,
+    "stddev_ns": 47473222,
+    "avg_ts": 80.594928,
+    "stddev_ts": 2.413871,
+    "samples_ns": [ 1529730792, 1557930160, 1636596682, 1585818829, 1636556420 ],
+    "samples_ts": [ 83.6749, 82.1603, 78.2111, 80.7154, 78.213 ]
+  }
+]

baselines/llama_cpp/cmp_llamacpp.py

@@ -0,0 +1,106 @@
+"""Greedy-decode parity for llama.cpp against the same fp32 reference vLLM is
+scored on.
+
+The point of the comparison is that "4-bit" is not one thing: llama.cpp's
+Q4_K_M and our `cpu_int4` spend a similar number of bits per weight but pick
+different grids, so the speed comparison is only meaningful next to a quality
+comparison run on identical prompts.  llama-server takes a prompt as raw token
+ids, so both engines see exactly the token sequence the reference did.
+"""
+import argparse, json, subprocess, time, urllib.request
+
+import numpy as np
+
+
+def post(url, payload, timeout=600):
+    req = urllib.request.Request(
+        url, data=json.dumps(payload).encode(),
+        headers={"Content-Type": "application/json"})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return json.loads(r.read())
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--gguf", required=True)
+    ap.add_argument("--ref", default="ref.json")
+    ap.add_argument("--ref-logits", default="ref_logits.npy")
+    ap.add_argument("--bin", default="/data/zhoutaichang/embedding_infer/llama.cpp/build/bin/llama-server")
+    ap.add_argument("--cores", default="56-87")
+    ap.add_argument("--threads", type=int, default=32)
+    ap.add_argument("--port", type=int, default=18099)
+    ap.add_argument("--ctx", type=int, default=4096)
+    ap.add_argument("--out", default="parity_llamacpp.json")
+    a = ap.parse_args()
+
+    ref = json.load(open(a.ref))
+    ref_logits = np.load(a.ref_logits)
+    n_new = len(ref["new_ids"])
+
+    proc = subprocess.Popen(
+        ["taskset", "-c", a.cores, a.bin, "-m", a.gguf, "--port", str(a.port),
+         "-t", str(a.threads), "-c", str(a.ctx), "--no-warmup", "--host", "127.0.0.1"],
+        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    url = f"http://127.0.0.1:{a.port}"
+    try:
+        for _ in range(180):
+            try:
+                urllib.request.urlopen(url + "/health", timeout=2)
+                break
+            except Exception:
+                time.sleep(1)
+        else:
+            raise RuntimeError("llama-server did not become healthy")
+
+        res = post(url + "/completion", {
+            "prompt": ref["prompt_ids"], "n_predict": n_new,
+            "temperature": 0.0, "top_k": 1, "n_probs": 20,
+            "cache_prompt": False, "post_sampling_probs": False,
+        })
+    finally:
+        proc.terminate()
+        try:
+            proc.wait(timeout=30)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+
+    probs = res.get("completion_probabilities") or []
+    ids = [p["id"] for p in probs] if probs and "id" in probs[0] else []
+    n = min(n_new, len(ids)) if ids else 0
+    match = sum(int(x == y) for x, y in zip(ref["new_ids"][:n], ids[:n]))
+
+    rep = {
+        "engine": "llama.cpp", "gguf": a.gguf.split("/")[-1],
+        "ref_text": ref["text"], "llamacpp_text": res.get("content", ""),
+        "greedy_token_match": f"{match}/{n}" if n else "n/a",
+        "exact_match": bool(n) and ref["new_ids"][:n] == ids[:n],
+        "ref_top1": ref["top1"],
+        "llamacpp_first_token": ids[0] if ids else None,
+        "first_token_agree": bool(ids) and ids[0] == ref["top1"],
+    }
+    if probs:
+        import math
+        top = (probs[0].get("top_logprobs") or probs[0].get("top_probs")
+               or probs[0].get("probs") or [])
+        first = {}
+        for e in top:
+            if "id" not in e:
+                continue
+            first[int(e["id"])] = (float(e["prob"]) if "prob" in e
+                                   else math.exp(float(e["logprob"])))
+        ref_top = np.argsort(-ref_logits)[:20]
+        ref_p = np.exp(ref_logits - ref_logits.max())
+        ref_p /= ref_p.sum()
+        pairs = [(int(t), float(ref_p[t]), first[int(t)]) for t in ref_top if int(t) in first]
+        rep["top20_prob_overlap"] = f"{len(pairs)}/20"
+        if pairs:
+            d = [abs(r - v) for _, r, v in pairs]
+            rep["top20_prob_max_abs_diff"] = max(d)
+            rep["top20_prob_mean_abs_diff"] = sum(d) / len(d)
+    print(json.dumps(rep, indent=2, ensure_ascii=False))
+    with open(a.out, "w") as f:
+        json.dump(rep, f, indent=2, ensure_ascii=False)
+
+
+if __name__ == "__main__":
+    main()

baselines/llama_cpp/il_llamacpp_p1.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T19:04:49Z",
+    "avg_ns": 489021600,
+    "stddev_ns": 23921490,
+    "avg_ts": 1048.939735,
+    "stddev_ts": 49.910229,
+    "samples_ns": [ 527122937, 485801934, 462083348, 490951261, 479148522 ],
+    "samples_ts": [ 971.31, 1053.93, 1108.03, 1042.87, 1068.56 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T19:04:52Z",
+    "avg_ns": 1570942064,
+    "stddev_ns": 20220055,
+    "avg_ts": 81.490699,
+    "stddev_ts": 1.061319,
+    "samples_ns": [ 1586700558, 1582528379, 1537750061, 1565679809, 1582051516 ],
+    "samples_ts": [ 80.6705, 80.8832, 83.2385, 81.7536, 80.9076 ]
+  }
+]

baselines/llama_cpp/il_llamacpp_p2.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T19:14:20Z",
+    "avg_ns": 464927159,
+    "stddev_ns": 22146638,
+    "avg_ts": 1103.200971,
+    "stddev_ts": 51.267874,
+    "samples_ns": [ 453191976, 479254490, 448568607, 496875740, 446744984 ],
+    "samples_ts": [ 1129.76, 1068.33, 1141.41, 1030.44, 1146.07 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T19:14:23Z",
+    "avg_ns": 1422234679,
+    "stddev_ns": 12240631,
+    "avg_ts": 90.004494,
+    "stddev_ts": 0.767183,
+    "samples_ns": [ 1422285965, 1416723437, 1414194914, 1414611159, 1443357923 ],
+    "samples_ts": [ 89.996, 90.3493, 90.5109, 90.4842, 88.6821 ]
+  }
+]

baselines/llama_cpp/il_llamacpp_p3.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T19:20:46Z",
+    "avg_ns": 456731353,
+    "stddev_ns": 6199988,
+    "avg_ts": 1121.174997,
+    "stddev_ts": 15.277320,
+    "samples_ns": [ 459406023, 462620142, 461507407, 450329050, 449794146 ],
+    "samples_ts": [ 1114.48, 1106.74, 1109.41, 1136.95, 1138.3 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 24,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T19:20:49Z",
+    "avg_ns": 1506056809,
+    "stddev_ns": 24193761,
+    "avg_ts": 85.007660,
+    "stddev_ts": 1.362252,
+    "samples_ns": [ 1478402259, 1532453276, 1530685126, 1494657866, 1494085519 ],
+    "samples_ts": [ 86.58, 83.5262, 83.6227, 85.6383, 85.6711 ]
+  }
+]

baselines/llama_cpp/llamacpp_grid_t16.json

@@ -0,0 +1,272 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:32:06Z",
+    "avg_ns": 1498583726,
+    "stddev_ns": 7171856,
+    "avg_ts": 341.662152,
+    "stddev_ts": 1.628172,
+    "samples_ns": [ 1493166066, 1510351399, 1500283975, 1493415633, 1495701559 ],
+    "samples_ts": [ 342.896, 338.994, 341.269, 342.838, 342.314 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:32:18Z",
+    "avg_ns": 5062657211,
+    "stddev_ns": 186447127,
+    "avg_ts": 25.309483,
+    "stddev_ts": 0.893384,
+    "samples_ns": [ 4950579569, 4934896396, 5035243526, 5388250087, 5004316480 ],
+    "samples_ts": [ 25.8556, 25.9377, 25.4208, 23.7554, 25.5779 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:32:43Z",
+    "avg_ns": 560100709,
+    "stddev_ns": 15689530,
+    "avg_ts": 914.690155,
+    "stddev_ts": 25.391149,
+    "samples_ns": [ 566521408, 541347341, 583132866, 553334911, 556167020 ],
+    "samples_ts": [ 903.761, 945.788, 878.016, 925.299, 920.587 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:32:47Z",
+    "avg_ns": 1933840318,
+    "stddev_ns": 51036422,
+    "avg_ts": 66.225568,
+    "stddev_ts": 1.706730,
+    "samples_ns": [ 1896818220, 1923412956, 1937369806, 1892702916, 2018897692 ],
+    "samples_ts": [ 67.4814, 66.5484, 66.069, 67.6282, 63.4009 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:32:57Z",
+    "avg_ns": 635046952,
+    "stddev_ns": 3342779,
+    "avg_ts": 806.257449,
+    "stddev_ts": 4.239145,
+    "samples_ns": [ 634661823, 637237702, 639487443, 631413242, 632434552 ],
+    "samples_ts": [ 806.729, 803.468, 800.641, 810.879, 809.57 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:33:01Z",
+    "avg_ns": 1790378809,
+    "stddev_ns": 14069215,
+    "avg_ts": 71.496788,
+    "stddev_ts": 0.562779,
+    "samples_ns": [ 1803158642, 1794510633, 1803271959, 1776172087, 1774780727 ],
+    "samples_ts": [ 70.9865, 71.3286, 70.9821, 72.0651, 72.1216 ]
+  }
+]

baselines/llama_cpp/llamacpp_grid_t32.json

@@ -0,0 +1,272 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:35:02Z",
+    "avg_ns": 1017103540,
+    "stddev_ns": 116210707,
+    "avg_ts": 509.454206,
+    "stddev_ts": 66.492143,
+    "samples_ns": [ 1045283058, 1099078888, 1103686903, 1017870763, 819598088 ],
+    "samples_ts": [ 489.819, 465.845, 463.9, 503.011, 624.696 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:35:08Z",
+    "avg_ns": 4481043455,
+    "stddev_ns": 53485519,
+    "avg_ts": 28.568042,
+    "stddev_ts": 0.342133,
+    "samples_ns": [ 4526208574, 4537101214, 4480636429, 4454908368, 4406362692 ],
+    "samples_ts": [ 28.2797, 28.2118, 28.5674, 28.7324, 29.0489 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:35:31Z",
+    "avg_ns": 343075990,
+    "stddev_ns": 8202776,
+    "avg_ts": 1493.062250,
+    "stddev_ts": 35.634573,
+    "samples_ns": [ 333407890, 348737139, 337165270, 342580383, 353489272 ],
+    "samples_ts": [ 1535.66, 1468.15, 1518.54, 1494.54, 1448.42 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:35:33Z",
+    "avg_ns": 1673245422,
+    "stddev_ns": 44193699,
+    "avg_ts": 76.540484,
+    "stddev_ts": 2.008808,
+    "samples_ns": [ 1727982320, 1650954337, 1650417455, 1711748955, 1625124045 ],
+    "samples_ts": [ 74.0748, 77.5309, 77.5561, 74.7773, 78.7632 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:35:42Z",
+    "avg_ns": 433558704,
+    "stddev_ns": 61442386,
+    "avg_ts": 1198.988860,
+    "stddev_ts": 159.663627,
+    "samples_ns": [ 384355143, 403569845, 520552799, 475384982, 383930753 ],
+    "samples_ts": [ 1332.1, 1268.68, 983.57, 1077.02, 1333.57 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:35:44Z",
+    "avg_ns": 1574917625,
+    "stddev_ns": 239068571,
+    "avg_ts": 82.814413,
+    "stddev_ts": 12.730560,
+    "samples_ns": [ 1383407468, 1295986814, 1572559050, 1776956669, 1845678125 ],
+    "samples_ts": [ 92.5252, 98.7664, 81.396, 72.0333, 69.3512 ]
+  }
+]

baselines/llama_cpp/llamacpp_grid_t4.json

@@ -0,0 +1,272 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:19:27Z",
+    "avg_ns": 7115257366,
+    "stddev_ns": 607044890,
+    "avg_ts": 72.394533,
+    "stddev_ts": 6.405320,
+    "samples_ns": [ 7765939766, 6729867579, 7316853262, 7495192180, 6268434044 ],
+    "samples_ts": [ 65.9289, 76.0788, 69.9754, 68.3105, 81.6791 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:20:09Z",
+    "avg_ns": 30700265663,
+    "stddev_ns": 4064662246,
+    "avg_ts": 4.196201,
+    "stddev_ts": 0.381497,
+    "samples_ns": [ 32683898432, 32626033248, 27736648629, 27745563194, 32709184813 ],
+    "samples_ts": [ 3.9163, 3.92325, 4.61483, 4.61335, 3.91327 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:22:44Z",
+    "avg_ns": 4412049470,
+    "stddev_ns": 642897239,
+    "avg_ts": 118.057116,
+    "stddev_ts": 17.378814,
+    "samples_ns": [ 4619378132, 3608885475, 4557507327, 3988149647, 5286326771 ],
+    "samples_ts": [ 110.837, 141.872, 112.342, 128.38, 96.8536 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:23:12Z",
+    "avg_ns": 13160648883,
+    "stddev_ns": 4037164376,
+    "avg_ts": 9.845522,
+    "stddev_ts": 1.260348,
+    "samples_ns": [ 12727171856, 13397387109, 13869421316, 15023258698, 10786005440 ],
+    "samples_ts": [ 10.0572, 9.5541, 9.22894, 8.52012, 11.8672 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:24:19Z",
+    "avg_ns": 4955359522,
+    "stddev_ns": 278495888,
+    "avg_ts": 103.589056,
+    "stddev_ts": 5.933882,
+    "samples_ns": [ 4969396818, 4806604145, 5198757330, 4567278377, 5234760941 ],
+    "samples_ts": [ 103.031, 106.52, 98.4851, 112.102, 97.8077 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:24:49Z",
+    "avg_ns": 12648866627,
+    "stddev_ns": 4253557830,
+    "avg_ts": 10.355908,
+    "stddev_ts": 1.831931,
+    "samples_ns": [ 12076753955, 12558032815, 15171316078, 9645037747, 13793192540 ],
+    "samples_ts": [ 10.5989, 10.1927, 8.43697, 13.2711, 9.27994 ]
+  }
+]

baselines/llama_cpp/llamacpp_grid_t8.json

@@ -0,0 +1,272 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:28:17Z",
+    "avg_ns": 2946090115,
+    "stddev_ns": 19481720,
+    "avg_ts": 173.795716,
+    "stddev_ts": 1.144798,
+    "samples_ns": [ 2976783425, 2943842928, 2936842142, 2948762947, 2924219135 ],
+    "samples_ts": [ 171.998, 173.922, 174.337, 173.632, 175.089 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B.gguf",
+    "model_type": "spark2_5 1.7B BF16",
+    "model_size": 3415547904,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:28:34Z",
+    "avg_ns": 7926507628,
+    "stddev_ns": 166210109,
+    "avg_ts": 16.153997,
+    "stddev_ts": 0.336782,
+    "samples_ns": [ 7746359669, 7836100448, 8141583693, 8059318007, 7849176324 ],
+    "samples_ts": [ 16.5239, 16.3347, 15.7218, 15.8822, 16.3074 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:29:15Z",
+    "avg_ns": 1044627273,
+    "stddev_ns": 8668625,
+    "avg_ts": 490.153690,
+    "stddev_ts": 4.025039,
+    "samples_ns": [ 1040367711, 1059967868, 1042470115, 1039018745, 1041311927 ],
+    "samples_ts": [ 492.134, 483.034, 491.141, 492.773, 491.687 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q8_0.gguf",
+    "model_type": "spark2_5 1.7B Q8_0",
+    "model_size": 1814728704,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:29:21Z",
+    "avg_ns": 2844703994,
+    "stddev_ns": 50284844,
+    "avg_ts": 45.007005,
+    "stddev_ts": 0.785784,
+    "samples_ns": [ 2841360663, 2852415251, 2800656934, 2925026017, 2804061106 ],
+    "samples_ts": [ 45.0488, 44.8743, 45.7036, 43.7603, 45.6481 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:29:36Z",
+    "avg_ns": 1179360692,
+    "stddev_ns": 9282489,
+    "avg_ts": 434.155009,
+    "stddev_ts": 3.414034,
+    "samples_ns": [ 1190805907, 1173513176, 1168400903, 1186783926, 1177299549 ],
+    "samples_ts": [ 429.961, 436.297, 438.206, 431.418, 434.894 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T06:29:43Z",
+    "avg_ns": 3018952734,
+    "stddev_ns": 27599225,
+    "avg_ts": 42.401651,
+    "stddev_ts": 0.388562,
+    "samples_ns": [ 3042267389, 2995063052, 3045385324, 2984815648, 3027232258 ],
+    "samples_ts": [ 42.0739, 42.737, 42.0308, 42.8837, 42.2828 ]
+  }
+]

baselines/llama_cpp/llamacpp_node0_warm_t32.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:47:23Z",
+    "avg_ns": 377579804,
+    "stddev_ns": 6351193,
+    "avg_ts": 1356.308108,
+    "stddev_ts": 22.545912,
+    "samples_ns": [ 372424973, 373894780, 373636992, 387431983, 380510292 ],
+    "samples_ts": [ 1374.77, 1369.37, 1370.31, 1321.52, 1345.56 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:47:25Z",
+    "avg_ns": 1305937293,
+    "stddev_ns": 13522740,
+    "avg_ts": 98.022226,
+    "stddev_ts": 1.005768,
+    "samples_ns": [ 1300442432, 1292366252, 1304438971, 1303902694, 1328536116 ],
+    "samples_ts": [ 98.428, 99.0431, 98.1265, 98.1668, 96.3466 ]
+  }
+]

baselines/llama_cpp/llamacpp_node1_t16.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:19:16Z",
+    "avg_ns": 640698554,
+    "stddev_ns": 8442449,
+    "avg_ts": 799.237980,
+    "stddev_ts": 10.456116,
+    "samples_ns": [ 652875468, 635459867, 646229581, 635186003, 633741853 ],
+    "samples_ts": [ 784.223, 805.716, 792.288, 806.063, 807.9 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:19:20Z",
+    "avg_ns": 1748941710,
+    "stddev_ns": 13313898,
+    "avg_ts": 73.190522,
+    "stddev_ts": 0.559300,
+    "samples_ns": [ 1759700788, 1762032558, 1728868153, 1744622542, 1749484513 ],
+    "samples_ts": [ 72.7396, 72.6434, 74.0369, 73.3683, 73.1644 ]
+  }
+]

baselines/llama_cpp/llamacpp_node1_t32.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:21:27Z",
+    "avg_ns": 379833208,
+    "stddev_ns": 2916251,
+    "avg_ts": 1348.023365,
+    "stddev_ts": 10.304461,
+    "samples_ns": [ 377554426, 384109520, 377555613, 381602693, 378343790 ],
+    "samples_ts": [ 1356.1, 1332.95, 1356.09, 1341.71, 1353.27 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:21:29Z",
+    "avg_ns": 1285969483,
+    "stddev_ns": 16587912,
+    "avg_ts": 99.548971,
+    "stddev_ts": 1.276520,
+    "samples_ns": [ 1279748126, 1273538113, 1309616758, 1270516087, 1296428332 ],
+    "samples_ts": [ 100.02, 100.507, 97.7385, 100.746, 98.7328 ]
+  }
+]

baselines/llama_cpp/llamacpp_node1_t4.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:14:01Z",
+    "avg_ns": 2437249430,
+    "stddev_ns": 66433948,
+    "avg_ts": 210.202021,
+    "stddev_ts": 5.922750,
+    "samples_ns": [ 2468232968, 2479407287, 2475612647, 2441614629, 2321379621 ],
+    "samples_ts": [ 207.436, 206.501, 206.817, 209.697, 220.558 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:14:15Z",
+    "avg_ns": 5289126007,
+    "stddev_ns": 63024171,
+    "avg_ts": 24.203349,
+    "stddev_ts": 0.288883,
+    "samples_ns": [ 5363916055, 5306624609, 5325596808, 5238325957, 5211166606 ],
+    "samples_ts": [ 23.8632, 24.1208, 24.0349, 24.4353, 24.5626 ]
+  }
+]

baselines/llama_cpp/llamacpp_node1_t8.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:16:53Z",
+    "avg_ns": 1253292754,
+    "stddev_ns": 79129486,
+    "avg_ts": 409.796891,
+    "stddev_ts": 25.209212,
+    "samples_ns": [ 1190927743, 1190669420, 1208414068, 1318310015, 1358142528 ],
+    "samples_ts": [ 429.917, 430.01, 423.696, 388.376, 376.985 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:17:01Z",
+    "avg_ns": 2909185423,
+    "stddev_ns": 35614966,
+    "avg_ts": 44.003861,
+    "stddev_ts": 0.540189,
+    "samples_ns": [ 2935902159, 2946667991, 2878606593, 2919344964, 2865405409 ],
+    "samples_ts": [ 43.5982, 43.4389, 44.466, 43.8455, 44.6708 ]
+  }
+]

baselines/llama_cpp/llamacpp_q4km_t16.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:19:51Z",
+    "avg_ns": 650543418,
+    "stddev_ns": 18555552,
+    "avg_ts": 787.546056,
+    "stddev_ts": 22.428007,
+    "samples_ns": [ 632032527, 670684809, 649115565, 632738336, 668145853 ],
+    "samples_ts": [ 810.085, 763.399, 788.766, 809.181, 766.3 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 16,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:19:55Z",
+    "avg_ns": 1781362880,
+    "stddev_ns": 21643541,
+    "avg_ts": 71.863545,
+    "stddev_ts": 0.869304,
+    "samples_ns": [ 1806353622, 1764741597, 1764660660, 1767373240, 1803685282 ],
+    "samples_ts": [ 70.861, 72.5319, 72.5352, 72.4239, 70.9658 ]
+  }
+]

baselines/llama_cpp/llamacpp_q4km_t32.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:21:20Z",
+    "avg_ns": 377608322,
+    "stddev_ns": 6410592,
+    "avg_ts": 1356.211504,
+    "stddev_ts": 22.766437,
+    "samples_ns": [ 372722694, 373942297, 373069875, 387378103, 380928645 ],
+    "samples_ts": [ 1373.68, 1369.2, 1372.4, 1321.71, 1344.08 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 32,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:21:22Z",
+    "avg_ns": 1342539204,
+    "stddev_ns": 21735221,
+    "avg_ts": 95.361608,
+    "stddev_ts": 1.535356,
+    "samples_ns": [ 1368242693, 1364047511, 1323244630, 1330069532, 1327091655 ],
+    "samples_ts": [ 93.5507, 93.8384, 96.7319, 96.2356, 96.4515 ]
+  }
+]

baselines/llama_cpp/llamacpp_q4km_t4.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:14:44Z",
+    "avg_ns": 2304834712,
+    "stddev_ns": 9004426,
+    "avg_ts": 222.144447,
+    "stddev_ts": 0.865236,
+    "samples_ns": [ 2319430963, 2305311298, 2304610521, 2298020765, 2296800014 ],
+    "samples_ts": [ 220.744, 222.096, 222.163, 222.8, 222.919 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 4,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:14:58Z",
+    "avg_ns": 5335792102,
+    "stddev_ns": 59840580,
+    "avg_ts": 23.991343,
+    "stddev_ts": 0.267723,
+    "samples_ns": [ 5425666503, 5348961684, 5341956156, 5275207674, 5287168494 ],
+    "samples_ts": [ 23.5916, 23.9299, 23.9613, 24.2644, 24.2096 ]
+  }
+]

baselines/llama_cpp/llamacpp_q4km_t8.json

@@ -0,0 +1,92 @@
+[
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 512,
+    "n_gen": 0,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:17:31Z",
+    "avg_ns": 1911672545,
+    "stddev_ns": 4048624965,
+    "avg_ts": 363.751611,
+    "stddev_ts": 143.367368,
+    "samples_ns": [ 4771767171, 1201291347, 1195946935, 1197665158, 1191692115 ],
+    "samples_ts": [ 107.298, 426.208, 428.113, 427.498, 429.641 ]
+  },
+  {
+    "build_commit": "e71b80510",
+    "build_number": 10849,
+    "cpu_info": "Intel(R) Xeon(R) Platinum 8480C",
+    "gpu_info": "",
+    "backends": "CPU",
+    "model_filename": "/data/zhoutaichang/.cache/huggingface/hub/models--XHToken--Spark-X2.5-1.7B-GGUF/snapshots/23e1fcac55e7dd71e4c12a23723cc228ba0e5e85/Spark-X2.5-1.7B-Q4_K_M.gguf",
+    "model_type": "spark2_5 1.7B Q4_K - Medium",
+    "model_size": 1102073856,
+    "model_n_params": 1707657216,
+    "n_batch": 2048,
+    "n_ubatch": 512,
+    "n_threads": 8,
+    "cpu_mask": "0x0",
+    "cpu_strict": false,
+    "poll": 50,
+    "type_k": "f16",
+    "type_v": "f16",
+    "n_gpu_layers": -1,
+    "n_cpu_moe": 0,
+    "split_mode": "layer",
+    "main_gpu": 0,
+    "no_kv_offload": false,
+    "flash_attn": -1,
+    "devices": "auto",
+    "tensor_split": "0.00",
+    "tensor_buft_overrides": "none",
+    "load_mode": "auto",
+    "lazy_mode": "auto",
+    "embeddings": false,
+    "no_op_offload": 0,
+    "no_host": false,
+    "fit_target": 0,
+    "fit_min_ctx": 0,
+    "n_prompt": 0,
+    "n_gen": 128,
+    "n_depth": 0,
+    "test_time": "2026-09-11T17:17:46Z",
+    "avg_ns": 2882701640,
+    "stddev_ns": 15288789,
+    "avg_ts": 44.403796,
+    "stddev_ts": 0.236813,
+    "samples_ns": [ 2894272938, 2890318648, 2890951071, 2881201632, 2856763912 ],
+    "samples_ts": [ 44.2253, 44.2858, 44.2761, 44.4259, 44.8059 ]
+  }
+]

baselines/quality_cmp_llamacpp-Q4_K_M.json

@@ -0,0 +1,80 @@
+{
+  "label": "llamacpp-Q4_K_M",
+  "prompts": 12,
+  "exact_match": "4/12",
+  "mean_prefix_kept": 0.5101,
+  "detail": [
+    {
+      "prompt": "chat_short",
+      "exact": false,
+      "common_prefix_chars": 91,
+      "ref_chars": 182
+    },
+    {
+      "prompt": "chat_long",
+      "exact": false,
+      "common_prefix_chars": 43,
+      "ref_chars": 246
+    },
+    {
+      "prompt": "chat_tools",
+      "exact": false,
+      "common_prefix_chars": 15,
+      "ref_chars": 290
+    },
+    {
+      "prompt": "plain_0",
+      "exact": true,
+      "common_prefix_chars": 361,
+      "ref_chars": 361
+    },
+    {
+      "prompt": "plain_1",
+      "exact": true,
+      "common_prefix_chars": 512,
+      "ref_chars": 512
+    },
+    {
+      "prompt": "plain_2",
+      "exact": false,
+      "common_prefix_chars": 71,
+      "ref_chars": 172
+    },
+    {
+      "prompt": "plain_3",
+      "exact": false,
+      "common_prefix_chars": 3,
+      "ref_chars": 352
+    },
+    {
+      "prompt": "plain_4",
+      "exact": true,
+      "common_prefix_chars": 318,
+      "ref_chars": 318
+    },
+    {
+      "prompt": "plain_5",
+      "exact": true,
+      "common_prefix_chars": 346,
+      "ref_chars": 346
+    },
+    {
+      "prompt": "plain_6",
+      "exact": false,
+      "common_prefix_chars": 2,
+      "ref_chars": 177
+    },
+    {
+      "prompt": "plain_7",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 320
+    },
+    {
+      "prompt": "plain_8",
+      "exact": false,
+      "common_prefix_chars": 2,
+      "ref_chars": 184
+    }
+  ]
+}

baselines/quality_cmp_vllm-W4A16-tinygemm.json

@@ -0,0 +1,80 @@
+{
+  "label": "vllm-W4A16-tinygemm",
+  "prompts": 12,
+  "exact_match": "2/12",
+  "mean_prefix_kept": 0.301,
+  "detail": [
+    {
+      "prompt": "chat_short",
+      "exact": false,
+      "common_prefix_chars": 91,
+      "ref_chars": 182
+    },
+    {
+      "prompt": "chat_long",
+      "exact": false,
+      "common_prefix_chars": 0,
+      "ref_chars": 261
+    },
+    {
+      "prompt": "chat_tools",
+      "exact": false,
+      "common_prefix_chars": 56,
+      "ref_chars": 287
+    },
+    {
+      "prompt": "plain_0",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 361
+    },
+    {
+      "prompt": "plain_1",
+      "exact": true,
+      "common_prefix_chars": 512,
+      "ref_chars": 512
+    },
+    {
+      "prompt": "plain_2",
+      "exact": false,
+      "common_prefix_chars": 4,
+      "ref_chars": 172
+    },
+    {
+      "prompt": "plain_3",
+      "exact": false,
+      "common_prefix_chars": 34,
+      "ref_chars": 352
+    },
+    {
+      "prompt": "plain_4",
+      "exact": true,
+      "common_prefix_chars": 318,
+      "ref_chars": 318
+    },
+    {
+      "prompt": "plain_5",
+      "exact": false,
+      "common_prefix_chars": 5,
+      "ref_chars": 346
+    },
+    {
+      "prompt": "plain_6",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 283
+    },
+    {
+      "prompt": "plain_7",
+      "exact": false,
+      "common_prefix_chars": 54,
+      "ref_chars": 320
+    },
+    {
+      "prompt": "plain_8",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 184
+    }
+  ]
+}

baselines/quality_cmp_vllm-W4A16.json

@@ -0,0 +1,80 @@
+{
+  "label": "vllm-W4A16",
+  "prompts": 12,
+  "exact_match": "2/12",
+  "mean_prefix_kept": 0.301,
+  "detail": [
+    {
+      "prompt": "chat_short",
+      "exact": false,
+      "common_prefix_chars": 91,
+      "ref_chars": 182
+    },
+    {
+      "prompt": "chat_long",
+      "exact": false,
+      "common_prefix_chars": 0,
+      "ref_chars": 261
+    },
+    {
+      "prompt": "chat_tools",
+      "exact": false,
+      "common_prefix_chars": 56,
+      "ref_chars": 287
+    },
+    {
+      "prompt": "plain_0",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 361
+    },
+    {
+      "prompt": "plain_1",
+      "exact": true,
+      "common_prefix_chars": 512,
+      "ref_chars": 512
+    },
+    {
+      "prompt": "plain_2",
+      "exact": false,
+      "common_prefix_chars": 4,
+      "ref_chars": 172
+    },
+    {
+      "prompt": "plain_3",
+      "exact": false,
+      "common_prefix_chars": 34,
+      "ref_chars": 352
+    },
+    {
+      "prompt": "plain_4",
+      "exact": true,
+      "common_prefix_chars": 318,
+      "ref_chars": 318
+    },
+    {
+      "prompt": "plain_5",
+      "exact": false,
+      "common_prefix_chars": 5,
+      "ref_chars": 346
+    },
+    {
+      "prompt": "plain_6",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 283
+    },
+    {
+      "prompt": "plain_7",
+      "exact": false,
+      "common_prefix_chars": 54,
+      "ref_chars": 320
+    },
+    {
+      "prompt": "plain_8",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 184
+    }
+  ]
+}

baselines/quality_cmp_vllm-W4A8-gptq.json

@@ -0,0 +1,80 @@
+{
+  "label": "vllm-W4A8-gptq",
+  "prompts": 12,
+  "exact_match": "1/12",
+  "mean_prefix_kept": 0.1263,
+  "detail": [
+    {
+      "prompt": "chat_short",
+      "exact": false,
+      "common_prefix_chars": 66,
+      "ref_chars": 182
+    },
+    {
+      "prompt": "chat_long",
+      "exact": false,
+      "common_prefix_chars": 14,
+      "ref_chars": 261
+    },
+    {
+      "prompt": "chat_tools",
+      "exact": false,
+      "common_prefix_chars": 16,
+      "ref_chars": 287
+    },
+    {
+      "prompt": "plain_0",
+      "exact": false,
+      "common_prefix_chars": 25,
+      "ref_chars": 361
+    },
+    {
+      "prompt": "plain_1",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 512
+    },
+    {
+      "prompt": "plain_2",
+      "exact": false,
+      "common_prefix_chars": 4,
+      "ref_chars": 172
+    },
+    {
+      "prompt": "plain_3",
+      "exact": false,
+      "common_prefix_chars": 4,
+      "ref_chars": 352
+    },
+    {
+      "prompt": "plain_4",
+      "exact": true,
+      "common_prefix_chars": 318,
+      "ref_chars": 318
+    },
+    {
+      "prompt": "plain_5",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 346
+    },
+    {
+      "prompt": "plain_6",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 283
+    },
+    {
+      "prompt": "plain_7",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 320
+    },
+    {
+      "prompt": "plain_8",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 184
+    }
+  ]
+}

baselines/quality_cmp_vllm-W4A8.json

@@ -0,0 +1,80 @@
+{
+  "label": "vllm-W4A8",
+  "prompts": 12,
+  "exact_match": "1/12",
+  "mean_prefix_kept": 0.1263,
+  "detail": [
+    {
+      "prompt": "chat_short",
+      "exact": false,
+      "common_prefix_chars": 66,
+      "ref_chars": 182
+    },
+    {
+      "prompt": "chat_long",
+      "exact": false,
+      "common_prefix_chars": 14,
+      "ref_chars": 261
+    },
+    {
+      "prompt": "chat_tools",
+      "exact": false,
+      "common_prefix_chars": 16,
+      "ref_chars": 287
+    },
+    {
+      "prompt": "plain_0",
+      "exact": false,
+      "common_prefix_chars": 25,
+      "ref_chars": 361
+    },
+    {
+      "prompt": "plain_1",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 512
+    },
+    {
+      "prompt": "plain_2",
+      "exact": false,
+      "common_prefix_chars": 4,
+      "ref_chars": 172
+    },
+    {
+      "prompt": "plain_3",
+      "exact": false,
+      "common_prefix_chars": 4,
+      "ref_chars": 352
+    },
+    {
+      "prompt": "plain_4",
+      "exact": true,
+      "common_prefix_chars": 318,
+      "ref_chars": 318
+    },
+    {
+      "prompt": "plain_5",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 346
+    },
+    {
+      "prompt": "plain_6",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 283
+    },
+    {
+      "prompt": "plain_7",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 320
+    },
+    {
+      "prompt": "plain_8",
+      "exact": false,
+      "common_prefix_chars": 1,
+      "ref_chars": 184
+    }
+  ]
+}

baselines/quality_llamacpp_bf16.json

@@ -0,0 +1,50 @@
+{
+  "chat_short": {
+    "text": "We are asked: \"What is the capital of France? Answer in one word.\" So the answer should be \"Paris\". But we need to output it in one word. So the final answer is \"Paris\".</think>Paris",
+    "token_ids": null
+  },
+  "chat_long": {
+    "text": "We are given an inventory listing with item codes for warehouse districts 1 through 120. The question asks: \"What is the code for warehouse district 73?\" We need to find the code corresponding to district 73.\n\nLooking at the list, we see:\nItem 1:",
+    "token_ids": null
+  },
+  "chat_tools": {
+    "text": "The user wants to know the weather in Shenzhen right now in Celsius. I have a function available to get weather for a city. Let me call it with the city \"Shenzhen\" and unit \"celsius\".</think><tool_call>get_weather<arg_key>city</arg_key><arg_value>Shenzhen</arg_value><arg_key>unit</arg_key>",
+    "token_ids": null
+  },
+  "plain_0": {
+    "text": " the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia",
+    "token_ids": null
+  },
+  "plain_1": {
+    "text": " mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission",
+    "token_ids": null
+  },
+  "plain_2": {
+    "text": "    a, b = 0, 1\n    for _ in range(n):\n        a, b = b, a + b\n        if _ == n:\n            return a\n    return a\n\n\ndef main():\n    n = int(input())\n    print(fibonacci(n",
+    "token_ids": null
+  },
+  "plain_3": {
+    "text": " the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that",
+    "token_ids": null
+  },
+  "plain_4": {
+    "text": " to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to",
+    "token_ids": null
+  },
+  "plain_5": {
+    "text": " the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the",
+    "token_ids": null
+  },
+  "plain_6": {
+    "text": " carbon and oxygen into glucose and other organic compounds.\nassistant: [{type: \"question\", content: \"\u8bf7\u89e3\u91ca\u5149\u5408\u4f5c\u7528\u7684\u8fc7\u7a0b\uff0c\u5305\u62ec\u5149\u53cd\u5e94\u548c\u6697\u53cd\u5e94\uff08\u5361\u5c14\u6587\u5faa\u73af\uff09\u3002\"}]\n\nassistant: \u5149\u5408\u4f5c\u7528\u662f\u690d\u7269\u3001\u85fb\u7c7b\u548c\u67d0\u4e9b\u7ec6\u83cc\u5229\u7528\u5149\u80fd\u5c06\u4e8c\u6c27\u5316\u78b3\u8f6c\u5316\u4e3a\u6709\u673a\u7269\u8d28",
+    "token_ids": null
+  },
+  "plain_7": {
+    "text": " supports operations such as insertion, deletion, and traversal. A binary search tree is a tree where each node has at most two children, and the left and right children are such that the left subtree contains all nodes with values less than the node's value, and the right subtree contains all nodes with values greater",
+    "token_ids": null
+  },
+  "plain_8": {
+    "text": " 1333. The Treaty of Westphalia was signed in 1333. The Treaty of Westphalia was signed in 1333. The Treaty of Westphalia was signed in 1333. The Treaty of Westphalia was signed in 133",
+    "token_ids": null
+  }
+}

baselines/quality_llamacpp_q4km.json

@@ -0,0 +1,50 @@
+{
+  "chat_short": {
+    "text": "We are asked: \"What is the capital of France? Answer in one word.\" So the answer should be a single word: \"Paris\". But we need to output just the answer in one word. The instruction says: \"Answer in one word.\" So we should respond with \"Paris\".</think>Paris",
+    "token_ids": null
+  },
+  "chat_long": {
+    "text": "We are given an inventory listing with items 1 through 120. Each item has a code for a warehouse district. The codes are given as:\n\nItem 1: 1007\nItem 2: 1014\nItem 3: 1021\n",
+    "token_ids": null
+  },
+  "chat_tools": {
+    "text": "The user wants weather for Shenzhen in celsius. I have the weather function available. Let me call it with the city \"Shenzhen\" and unit \"celsius\".</think><tool_call>get_weather<arg_key>city</arg_key><arg_value>Shenzhen</arg_value><arg_key>unit</arg_key><arg_value>celsius</arg_value></tool_call>",
+    "token_ids": null
+  },
+  "plain_0": {
+    "text": " the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia",
+    "token_ids": null
+  },
+  "plain_1": {
+    "text": " mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission",
+    "token_ids": null
+  },
+  "plain_2": {
+    "text": "    a, b = 0, 1\n    for _ in range(n):\n        a, b = b, a + b\n        return b\n\n\nfib = fibonacci\n\n\nprint(fib(10))\nprint(fib(20))\nprint(fib",
+    "token_ids": null
+  },
+  "plain_3": {
+    "text": " that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that that",
+    "token_ids": null
+  },
+  "plain_4": {
+    "text": " to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to",
+    "token_ids": null
+  },
+  "plain_5": {
+    "text": " the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the",
+    "token_ids": null
+  },
+  "plain_6": {
+    "text": " converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts converts",
+    "token_ids": null
+  },
+  "plain_7": {
+    "text": " is used to organize data in a way that is used to organize data in a way that is used to organize data in a way that is used to organize data in a way that is used to organize data in a way that is used to organize data in a way that is used to organize data in a way that is",
+    "token_ids": null
+  },
+  "plain_8": {
+    "text": " 1621. The Treaty of Westphalia was signed in 1621. The Treaty of Westphalia was signed in 1621. The Treaty of Westphalia was signed in 1621. The Treaty of Westphalia was signed in 162",
+    "token_ids": null
+  }
+}

baselines/quality_vllm_bf16.json

@@ -0,0 +1,812 @@
+{
+  "chat_short": {
+    "text": "We are asked: \"What is the capital of France? Answer in one word.\" So the answer should be \"Paris\". But we need to output it in one word. So the final answer is \"Paris\".</think>Paris",
+    "token_ids": [
+      3066,
+      599,
+      5509,
+      39,
+      518,
+      4766,
+      390,
+      275,
+      11862,
+      314,
+      17349,
+      44,
+      74288,
+      304,
+      1135,
+      5240,
+      1848,
+      4358,
+      275,
+      6787,
+      2029,
+      417,
+      518,
+      110949,
+      4348,
+      2964,
+      679,
+      1633,
+      312,
+      6340,
+      488,
+      304,
+      1135,
+      5240,
+      27,
+      4358,
+      275,
+      4625,
+      6787,
+      390,
+      518,
+      110949,
+      4348,
+      4,
+      110949,
+      1
+    ]
+  },
+  "chat_long": {
+    "text": "We are given an inventory listing with item codes for warehouse districts 1 through 120. The question asks: \"What is the code for warehouse district 73?\" We need to find the code corresponding to district 73.\n\nLooking at the list, we see that item 1 is district",
+    "token_ids": [
+      3066,
+      599,
+      4114,
+      300,
+      31946,
+      35404,
+      478,
+      7006,
+      27149,
+      396,
+      56774,
+      49408,
+      220,
+      30,
+      1953,
+      220,
+      30,
+      31,
+      29,
+      27,
+      614,
+      5565,
+      27179,
+      39,
+      518,
+      4766,
+      390,
+      275,
+      5760,
+      396,
+      56774,
+      20882,
+      220,
+      36,
+      32,
+      5171,
+      1994,
+      1633,
+      312,
+      2432,
+      275,
+      5760,
+      12610,
+      312,
+      20882,
+      220,
+      36,
+      32,
+      27,
+      198,
+      198,
+      48992,
+      511,
+      275,
+      2631,
+      25,
+      679,
+      2261,
+      455,
+      7006,
+      220,
+      30,
+      390,
+      20882
+    ]
+  },
+  "chat_tools": {
+    "text": "The user wants to know the weather in Shenzhen right now in Celsius. I have a function available called `get_weather` that can retrieve current weather for a city. Let me call it with the city \"Shenzhen\" and unit \"celsius\".</think><tool_call>get_weather<arg_key>city</arg_key><arg_value>",
+    "token_ids": [
+      1046,
+      3754,
+      13025,
+      312,
+      1843,
+      275,
+      14510,
+      304,
+      75040,
+      103,
+      872,
+      2430,
+      2343,
+      304,
+      423,
+      108364,
+      27,
+      373,
+      760,
+      259,
+      2122,
+      4234,
+      4848,
+      2561,
+      913,
+      76,
+      64301,
+      77,
+      455,
+      764,
+      41002,
+      3345,
+      14510,
+      396,
+      259,
+      6594,
+      27,
+      9619,
+      713,
+      2426,
+      488,
+      478,
+      275,
+      6594,
+      518,
+      64,
+      872,
+      103,
+      872,
+      15,
+      317,
+      9261,
+      518,
+      8281,
+      4348,
+      4,
+      130977,
+      913,
+      76,
+      64301,
+      130980,
+      58593,
+      130981,
+      130982
+    ]
+  },
+  "plain_0": {
+    "text": " the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia is the capital of Australia",
+    "token_ids": [
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686,
+      390,
+      275,
+      11862,
+      314,
+      14686
+    ]
+  },
+  "plain_1": {
+    "text": " mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission",
+    "token_ids": [
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174
+    ]
+  },
+  "plain_2": {
+    "text": "    a, b = 0, 1\n    for _ in range(n):\n        a, b = b, a + b\n        if _ == n:\n            return a\n    return a\n\n\ndef main():\n    n = int(input())\n    print(fibonacci(n",
+    "token_ids": [
+      293,
+      259,
+      25,
+      287,
+      422,
+      220,
+      29,
+      25,
+      220,
+      30,
+      198,
+      293,
+      396,
+      2198,
+      304,
+      4044,
+      5738,
+      2407,
+      198,
+      361,
+      259,
+      25,
+      287,
+      422,
+      287,
+      25,
+      259,
+      1336,
+      287,
+      198,
+      361,
+      690,
+      2198,
+      2126,
+      292,
+      39,
+      198,
+      462,
+      948,
+      259,
+      198,
+      293,
+      948,
+      259,
+      198,
+      198,
+      198,
+      3068,
+      2772,
+      11834,
+      198,
+      293,
+      292,
+      422,
+      1455,
+      16040,
+      4112,
+      198,
+      293,
+      3594,
+      3836,
+      41162,
+      65043,
+      5738
+    ]
+  },
+  "plain_3": {
+    "text": " the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that the three laws of thermodynamics state that",
+    "token_ids": [
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      3519,
+      455
+    ]
+  },
+  "plain_4": {
+    "text": " to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to",
+    "token_ids": [
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312
+    ]
+  },
+  "plain_5": {
+    "text": " the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the difference between TCP and UDP is the",
+    "token_ids": [
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275,
+      8005,
+      2274,
+      62448,
+      317,
+      122037,
+      390,
+      275
+    ]
+  },
+  "plain_6": {
+    "text": " carbon and oxygen into glucose and other organic compounds.\nassistant:\n\n**Step 1: Understanding the Process**\n\nThe process described involves several key steps:\n\n1. **Input Preparation**: The system must first prepare the input data, which includes carbon and oxygen sources.\n2. **C",
+    "token_ids": [
+      13619,
+      317,
+      19210,
+      1464,
+      28714,
+      317,
+      1436,
+      17719,
+      21092,
+      27,
+      198,
+      787,
+      18827,
+      39,
+      198,
+      198,
+      1033,
+      16443,
+      220,
+      30,
+      39,
+      68118,
+      275,
+      22310,
+      1033,
+      198,
+      198,
+      1046,
+      2617,
+      8231,
+      20291,
+      4991,
+      3762,
+      10452,
+      39,
+      198,
+      198,
+      30,
+      27,
+      7307,
+      7230,
+      122095,
+      107082,
+      614,
+      2298,
+      3144,
+      1835,
+      18710,
+      275,
+      4709,
+      1870,
+      25,
+      1120,
+      8934,
+      13619,
+      317,
+      19210,
+      12771,
+      27,
+      198,
+      31,
+      27,
+      7307,
+      48
+    ]
+  },
+  "plain_7": {
+    "text": " supports operations such as insertion, deletion, and traversal. A binary search tree is a tree where each node has at most two children, and the left and right children are such that the left subtree contains all nodes with values less than the node's value, and the right subtree contains all nodes with values greater",
+    "token_ids": [
+      20815,
+      13127,
+      2129,
+      463,
+      54525,
+      25,
+      62316,
+      25,
+      317,
+      8709,
+      46267,
+      27,
+      401,
+      20470,
+      6258,
+      9905,
+      390,
+      259,
+      9905,
+      2067,
+      2440,
+      6795,
+      1044,
+      511,
+      2026,
+      1841,
+      4672,
+      25,
+      317,
+      275,
+      3887,
+      317,
+      2430,
+      4672,
+      599,
+      2129,
+      455,
+      275,
+      3887,
+      1733,
+      33440,
+      10736,
+      847,
+      15325,
+      478,
+      4906,
+      3921,
+      1489,
+      275,
+      6795,
+      1045,
+      2613,
+      25,
+      317,
+      275,
+      2430,
+      1733,
+      33440,
+      10736,
+      847,
+      15325,
+      478,
+      4906,
+      9063
+    ]
+  },
+  "plain_8": {
+    "text": " 1333. The Treaty of Westphalia was signed in 1333. The Treaty of Westphalia was signed in 1333. The Treaty of Westphalia was signed in 1333. The Treaty of Westphalia was signed in 133",
+    "token_ids": [
+      220,
+      30,
+      32,
+      32,
+      32,
+      27,
+      614,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      220,
+      30,
+      32,
+      32,
+      32,
+      27,
+      614,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      220,
+      30,
+      32,
+      32,
+      32,
+      27,
+      614,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      220,
+      30,
+      32,
+      32,
+      32,
+      27,
+      614,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      220,
+      30,
+      32,
+      32
+    ]
+  }
+}

baselines/quality_vllm_w4a16.json

@@ -0,0 +1,817 @@
+{
+  "chat_short": {
+    "text": "We are asked: \"What is the capital of France? Answer in one word.\" So the answer should be just \"Paris\". But the instruction says \"Answer in one word.\" So I should output \"Paris\" as the final answer.</think>Paris",
+    "token_ids": [
+      3066,
+      599,
+      5509,
+      39,
+      518,
+      4766,
+      390,
+      275,
+      11862,
+      314,
+      17349,
+      44,
+      74288,
+      304,
+      1135,
+      5240,
+      1848,
+      4358,
+      275,
+      6787,
+      2029,
+      417,
+      1839,
+      518,
+      110949,
+      4348,
+      2964,
+      275,
+      23220,
+      5685,
+      518,
+      47648,
+      304,
+      1135,
+      5240,
+      1848,
+      4358,
+      373,
+      2029,
+      6340,
+      518,
+      110949,
+      15,
+      463,
+      275,
+      4625,
+      6787,
+      27,
+      4,
+      110949,
+      1
+    ]
+  },
+  "chat_long": {
+    "text": "Got it, let's see. The user is asking for the code of warehouse district 73. Let's look through the inventory list. Let's check each item's district number and code.\nWait, let's list them out properly. Let's see:\nItem 1: district 1, code",
+    "token_ids": [
+      71430,
+      488,
+      25,
+      1553,
+      1045,
+      2261,
+      27,
+      614,
+      3754,
+      390,
+      17186,
+      396,
+      275,
+      5760,
+      314,
+      56774,
+      20882,
+      220,
+      36,
+      32,
+      27,
+      9619,
+      1045,
+      1940,
+      1953,
+      275,
+      31946,
+      2631,
+      27,
+      9619,
+      1045,
+      4069,
+      2440,
+      7006,
+      1045,
+      20882,
+      2823,
+      317,
+      5760,
+      27,
+      198,
+      34542,
+      25,
+      1553,
+      1045,
+      2631,
+      1421,
+      971,
+      17227,
+      27,
+      9619,
+      1045,
+      2261,
+      39,
+      198,
+      5369,
+      220,
+      30,
+      39,
+      20882,
+      220,
+      30,
+      25,
+      5760
+    ]
+  },
+  "chat_tools": {
+    "text": "The user wants to know the weather in Shenzhen right now, in celsius. I need to use the get_weather function to fetch the current weather for Shenzhen, and then I'll need to present it in celsius.\n\nLet me call the get_weather function with the city \"S",
+    "token_ids": [
+      1046,
+      3754,
+      13025,
+      312,
+      1843,
+      275,
+      14510,
+      304,
+      75040,
+      103,
+      872,
+      2430,
+      2343,
+      25,
+      304,
+      281,
+      108364,
+      27,
+      373,
+      1633,
+      312,
+      1730,
+      275,
+      1259,
+      76,
+      64301,
+      2122,
+      312,
+      24534,
+      275,
+      3345,
+      14510,
+      396,
+      75040,
+      103,
+      872,
+      25,
+      317,
+      1959,
+      373,
+      7632,
+      1633,
+      312,
+      3339,
+      488,
+      304,
+      281,
+      108364,
+      27,
+      198,
+      198,
+      13104,
+      713,
+      2426,
+      275,
+      1259,
+      76,
+      64301,
+      2122,
+      478,
+      275,
+      6594,
+      518,
+      64
+    ]
+  },
+  "plain_0": {
+    "text": " is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is",
+    "token_ids": [
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390
+    ]
+  },
+  "plain_1": {
+    "text": " mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission mission",
+    "token_ids": [
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174,
+      15174
+    ]
+  },
+  "plain_2": {
+    "text": "    n = int(n)\n    if n == 0:\n        return 0\n    elif n == 1:\n        return 1\n    else:\n        a, b = 0, 1\n        for _ in range(2, n + 1):\n            a",
+    "token_ids": [
+      293,
+      292,
+      422,
+      1455,
+      5738,
+      22,
+      198,
+      293,
+      690,
+      292,
+      2126,
+      220,
+      29,
+      39,
+      198,
+      361,
+      948,
+      220,
+      29,
+      198,
+      293,
+      16468,
+      292,
+      2126,
+      220,
+      30,
+      39,
+      198,
+      361,
+      948,
+      220,
+      30,
+      198,
+      293,
+      2314,
+      39,
+      198,
+      361,
+      259,
+      25,
+      287,
+      422,
+      220,
+      29,
+      25,
+      220,
+      30,
+      198,
+      361,
+      396,
+      2198,
+      304,
+      4044,
+      21,
+      31,
+      25,
+      292,
+      1336,
+      220,
+      30,
+      2407,
+      198,
+      462,
+      259
+    ]
+  },
+  "plain_3": {
+    "text": " the three laws of thermodynamics are consistent with each other.\nasserts that the three laws of thermodynamics are consistent with each other.\nassertThat(assertThat(assertThat(assertThat(assertThat(assertThat(assertThat(assertThat(assertThat(assertThat(assertThat(assertThat",
+    "token_ids": [
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      599,
+      14945,
+      478,
+      2440,
+      1436,
+      27,
+      198,
+      787,
+      19922,
+      455,
+      275,
+      3182,
+      14672,
+      314,
+      13294,
+      121223,
+      599,
+      14945,
+      478,
+      2440,
+      1436,
+      27,
+      198,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837,
+      21,
+      13462,
+      7837
+    ]
+  },
+  "plain_4": {
+    "text": " to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to",
+    "token_ids": [
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312
+    ]
+  },
+  "plain_5": {
+    "text": " the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the the",
+    "token_ids": [
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275,
+      275
+    ]
+  },
+  "plain_6": {
+    "text": " glucose into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into energy into",
+    "token_ids": [
+      28714,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464,
+      5404,
+      1464
+    ]
+  },
+  "plain_7": {
+    "text": " supports operations such as insertion, deletion, and searching. A binary search tree is a data structure that supports operations such as insertion, deletion, and searching. A binary search tree is a data structure that supports operations such as insertion, deletion, and searching. A binary search tree is a data structure that supports operations such as",
+    "token_ids": [
+      20815,
+      13127,
+      2129,
+      463,
+      54525,
+      25,
+      62316,
+      25,
+      317,
+      25124,
+      27,
+      401,
+      20470,
+      6258,
+      9905,
+      390,
+      259,
+      1870,
+      6982,
+      455,
+      20815,
+      13127,
+      2129,
+      463,
+      54525,
+      25,
+      62316,
+      25,
+      317,
+      25124,
+      27,
+      401,
+      20470,
+      6258,
+      9905,
+      390,
+      259,
+      1870,
+      6982,
+      455,
+      20815,
+      13127,
+      2129,
+      463,
+      54525,
+      25,
+      62316,
+      25,
+      317,
+      25124,
+      27,
+      401,
+      20470,
+      6258,
+      9905,
+      390,
+      259,
+      1870,
+      6982,
+      455,
+      20815,
+      13127,
+      2129,
+      463
+    ]
+  },
+  "plain_8": {
+    "text": " the Treaty of Westphalia was signed in the Treaty of Westphalia was signed in the Treaty of Westphalia was signed in the Treaty of Westphalia was signed in the Treaty of Westphalia was signed in the Treaty of Westphalia was signed in the Treaty of Westphalia was signed in the",
+    "token_ids": [
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275,
+      120455,
+      314,
+      10519,
+      1672,
+      14288,
+      572,
+      19750,
+      304,
+      275
+    ]
+  }
+}

baselines/quality_vllm_w4a8.json

@@ -0,0 +1,830 @@
+{
+  "chat_short": {
+    "text": "We are asked: \"What is the capital of France? Answer in one word.\"\n\nThe capital of France is Paris. But the instruction says \"Answer in one word.\" So we should answer with just \"Paris\" or \"PARIS\"? Since it says \"one word\", we should output \"Paris\" as",
+    "token_ids": [
+      3066,
+      599,
+      5509,
+      39,
+      518,
+      4766,
+      390,
+      275,
+      11862,
+      314,
+      17349,
+      44,
+      74288,
+      304,
+      1135,
+      5240,
+      1848,
+      198,
+      198,
+      1046,
+      11862,
+      314,
+      17349,
+      390,
+      19500,
+      27,
+      2964,
+      275,
+      23220,
+      5685,
+      518,
+      47648,
+      304,
+      1135,
+      5240,
+      1848,
+      4358,
+      679,
+      2029,
+      6787,
+      478,
+      1839,
+      518,
+      110949,
+      15,
+      553,
+      518,
+      80763,
+      4021,
+      32859,
+      11342,
+      488,
+      5685,
+      518,
+      841,
+      5240,
+      1078,
+      679,
+      2029,
+      6340,
+      518,
+      110949,
+      15,
+      463
+    ]
+  },
+  "chat_long": {
+    "text": "We are given a list of items, each with a code for a warehouse district. The list starts with Item 1 to Item 120, each with a code. We need to find the code for warehouse district 73.\n\nLet's parse the given data. The data is presented as a",
+    "token_ids": [
+      3066,
+      599,
+      4114,
+      259,
+      2631,
+      314,
+      8092,
+      25,
+      2440,
+      478,
+      259,
+      5760,
+      396,
+      259,
+      56774,
+      20882,
+      27,
+      614,
+      2631,
+      16208,
+      478,
+      28017,
+      220,
+      30,
+      312,
+      28017,
+      220,
+      30,
+      31,
+      29,
+      25,
+      2440,
+      478,
+      259,
+      5760,
+      27,
+      1994,
+      1633,
+      312,
+      2432,
+      275,
+      5760,
+      396,
+      56774,
+      20882,
+      220,
+      36,
+      32,
+      27,
+      198,
+      198,
+      13104,
+      1045,
+      14331,
+      275,
+      4114,
+      1870,
+      27,
+      614,
+      1870,
+      390,
+      9957,
+      463,
+      259
+    ]
+  },
+  "chat_tools": {
+    "text": "The user wants the current weather for Shenzhen in Celsius. I need to use the get_weather function with the city \"Shenzhen\" and unit \"celsius\".\n\nLet me call the function.</think>I'll get the current weather for Shenzhen right now.<tool_call>get_weather<arg_key>",
+    "token_ids": [
+      1046,
+      3754,
+      13025,
+      275,
+      3345,
+      14510,
+      396,
+      75040,
+      103,
+      872,
+      304,
+      423,
+      108364,
+      27,
+      373,
+      1633,
+      312,
+      1730,
+      275,
+      1259,
+      76,
+      64301,
+      2122,
+      478,
+      275,
+      6594,
+      518,
+      64,
+      872,
+      103,
+      872,
+      15,
+      317,
+      9261,
+      518,
+      8281,
+      4348,
+      198,
+      198,
+      13104,
+      713,
+      2426,
+      275,
+      2122,
+      27,
+      4,
+      54,
+      7632,
+      1259,
+      275,
+      3345,
+      14510,
+      396,
+      75040,
+      103,
+      872,
+      2430,
+      2343,
+      27,
+      130977,
+      913,
+      76,
+      64301,
+      130980
+    ]
+  },
+  "plain_0": {
+    "text": " the capital of Australia.\nassistant\nThe user is asking me to act as an assistant. I need to understand the context. The user's current message is: \"The user is asking me to act as an assistant. I need to understand the context. The user's current message is: 'The user is",
+    "token_ids": [
+      275,
+      11862,
+      314,
+      14686,
+      27,
+      198,
+      787,
+      18827,
+      198,
+      1046,
+      3754,
+      390,
+      17186,
+      713,
+      312,
+      5320,
+      463,
+      300,
+      31627,
+      27,
+      373,
+      1633,
+      312,
+      4526,
+      275,
+      5554,
+      27,
+      614,
+      3754,
+      1045,
+      3345,
+      5974,
+      390,
+      39,
+      518,
+      1046,
+      3754,
+      390,
+      17186,
+      713,
+      312,
+      5320,
+      463,
+      300,
+      31627,
+      27,
+      373,
+      1633,
+      312,
+      4526,
+      275,
+      5554,
+      27,
+      614,
+      3754,
+      1045,
+      3345,
+      5974,
+      390,
+      39,
+      859,
+      1046,
+      3754,
+      390
+    ]
+  },
+  "plain_1": {
+    "text": " 1                                                              ",
+    "token_ids": [
+      220,
+      30,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220,
+      220
+    ]
+  },
+  "plain_2": {
+    "text": "    return [1, 1] + [0] + [1] * (n - 2)\n\n\n# Test the Fibonacci function.\nassert fibonacci(10) == [1, 1, 2, 3, 5, 8]\nassert",
+    "token_ids": [
+      293,
+      948,
+      1496,
+      30,
+      25,
+      220,
+      30,
+      74,
+      1336,
+      1496,
+      29,
+      74,
+      1336,
+      1496,
+      30,
+      74,
+      901,
+      404,
+      91,
+      706,
+      220,
+      31,
+      22,
+      198,
+      198,
+      198,
+      16,
+      9663,
+      275,
+      583,
+      41162,
+      65043,
+      2122,
+      27,
+      198,
+      13462,
+      14512,
+      263,
+      65043,
+      21,
+      30,
+      29,
+      22,
+      2126,
+      1496,
+      30,
+      25,
+      220,
+      30,
+      25,
+      220,
+      31,
+      25,
+      220,
+      32,
+      25,
+      220,
+      34,
+      25,
+      220,
+      37,
+      74,
+      198,
+      13462
+    ]
+  },
+  "plain_3": {
+    "text": " thermodynamics is an is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is",
+    "token_ids": [
+      13294,
+      121223,
+      390,
+      300,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390
+    ]
+  },
+  "plain_4": {
+    "text": " to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to make a proper espresso you need to",
+    "token_ids": [
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312,
+      1996,
+      259,
+      3650,
+      128403,
+      454,
+      1633,
+      312
+    ]
+  },
+  "plain_5": {
+    "text": " is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is is",
+    "token_ids": [
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390,
+      390
+    ]
+  },
+  "plain_6": {
+    "text": " into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into into",
+    "token_ids": [
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464,
+      1464
+    ]
+  },
+  "plain_7": {
+    "text": " is used to represent a data structure that is used to represent a data structure that is used to represent a data structure that is used to represent a data structure that is used to represent a data structure that is used to represent a data structure that is used to represent a data structure that is used to represent a data structure that",
+    "token_ids": [
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455,
+      390,
+      2175,
+      312,
+      5207,
+      259,
+      1870,
+      6982,
+      455
+    ]
+  },
+  "plain_8": {
+    "text": " Westphalia. Westphalia was a region of Germany.\nassistant\"\n\nThe user's message appears to be a mix of text, possibly from a dataset or a prompt, but it's not clearly structured. I need to understand what the user wants me to do. Since the user's message is",
+    "token_ids": [
+      10519,
+      1672,
+      14288,
+      27,
+      10519,
+      1672,
+      14288,
+      572,
+      259,
+      6987,
+      314,
+      19356,
+      27,
+      198,
+      787,
+      18827,
+      15,
+      198,
+      198,
+      1046,
+      3754,
+      1045,
+      5974,
+      12718,
+      312,
+      417,
+      259,
+      12213,
+      314,
+      4217,
+      25,
+      16809,
+      692,
+      259,
+      23977,
+      553,
+      259,
+      20207,
+      25,
+      961,
+      488,
+      1045,
+      702,
+      12269,
+      42689,
+      27,
+      373,
+      1633,
+      312,
+      4526,
+      1568,
+      275,
+      3754,
+      13025,
+      713,
+      312,
+      793,
+      27,
+      11342,
+      275,
+      3754,
+      1045,
+      5974,
+      390
+    ]
+  }
+}

benchmarks/diffusion/README.md

@@ -17,18 +17,18 @@ The main entrypoint is:
 vllm serve Qwen/Qwen-Image --omni --port 8099
 ```
 
-2. Run a minimal benchmark:
+1. Run a minimal benchmark:
 
 ```bash
 python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-	--base-url http://localhost:8099 \
-	--model Qwen/Qwen-Image \
-	--task t2i \
-	--dataset vbench \
-	--num-prompts 5
+ --base-url http://localhost:8099 \
+ --model Qwen/Qwen-Image \
+ --task t2i \
+ --dataset vbench \
+ --num-prompts 5
 ```
 
-**Notes**
+### Notes
 
 - By default, image tasks talk to `http://<host>:<port>/v1/chat/completions`; video tasks talk to `/v1/videos`.
 - If you run the server on another host or port, pass `--base-url` accordingly.
@@ -52,14 +52,14 @@ Example (`t2v`):
 
 ```bash
 python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-	--base-url http://localhost:8099 \
-	--model Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-	--task t2v \
-	--dataset vbench \
-	--num-prompts 50 \
-	--width 640 --height 480 \
-	--num-frames 81 --fps 16 \
-	--num-inference-steps 40
+ --base-url http://localhost:8099 \
+ --model Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+ --task t2v \
+ --dataset vbench \
+ --num-prompts 50 \
+ --width 640 --height 480 \
+ --num-frames 81 --fps 16 \
+ --num-inference-steps 40
 ```
 
 Note: `vbench` can also be used for other tasks such as `t2i` / `i2v` (and `i2i`). For `t2i`, the loader reuses VBench t2v text prompts; for `i2v` / `i2i`, it loads the VBench i2v dataset (with image paths).
@@ -140,6 +140,21 @@ Traffic / concurrency flags:
 - `--request-rate`: Target request rate (requests/second). If set to `inf`, the script sends all requests immediately.
 - `--max-concurrency`: Max number of in-flight requests (default: `1`). This can hard-cap the achieved QPS: if it is too small, requests will queue behind the semaphore, and both achieved throughput and observed SLO attainment can be skewed.
 
+### Video job timeout and failures
+
+`--video-job-timeout` controls how long each `/v1/videos` job may be polled,
+including time queued on the server (default: `900` seconds). For long video
+jobs at high concurrency, use a larger budget, for example
+`--video-job-timeout 1800`. This also applies to warmup jobs. The polling budget
+starts after job creation; it excludes time waiting for the client concurrency
+semaphore. Individual HTTP calls still use the aiohttp session timeout.
+
+An expired job is counted as failed and deleted, which cancels queued or running
+server generation. The benchmark continues processing the remaining requests.
+The progress bar counts both successful and failed requests. The final report
+shows the failure count and the first 10 errors; `--output-file` saves all errors
+with their request IDs in `request_errors`.
+
 ### Batched warmup note
 
 For batched serving runs, warm the same in-flight shape you plan to measure.

benchmarks/diffusion/backends.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 import asyncio
 import base64
 import json
@@ -43,6 +46,7 @@ class RequestFuncInput:
     video_paths: list[str] | None = None
     request_id: str = field(default_factory=lambda: str(uuid.uuid4()))
     default_bot_task: str | None = DEFAULT_EDITS_BOT_TASK
+    video_job_timeout: float = 900.0
 
 
 @dataclass
@@ -331,64 +335,64 @@ async def async_request_v1_videos(
     output = RequestFuncOutput()
     output.start_time = time.perf_counter()
 
-    files = dict(input.extra_body)
-    if input.prompt:
-        files.setdefault("prompt", input.prompt)
-    if input.width and input.height:
-        files.setdefault("height", input.height)
-        files.setdefault("width", input.width)
-    if input.num_frames:
-        files.setdefault("num_frames", input.num_frames)
-    if input.num_inference_steps:
-        files.setdefault("num_inference_steps", input.num_inference_steps)
-    if input.seed is not None:
-        files.setdefault("seed", input.seed)
-    if input.fps:
-        files.setdefault("fps", input.fps)
-
-    form = aiohttp.FormData()
-    for k, v in files.items():
-        form.add_field(k, str(v))
-
     image_file = None
-    if input.image_paths and input.video_paths:
-        output.error = "Only one of image_paths or video_paths can be provided"
-        output.success = False
-        return output
+    job_id = None
+    try:
+        files = dict(input.extra_body)
+        if input.prompt:
+            files.setdefault("prompt", input.prompt)
+        if input.width and input.height:
+            files.setdefault("height", input.height)
+            files.setdefault("width", input.width)
+        if input.num_frames:
+            files.setdefault("num_frames", input.num_frames)
+        if input.num_inference_steps:
+            files.setdefault("num_inference_steps", input.num_inference_steps)
+        if input.seed is not None:
+            files.setdefault("seed", input.seed)
+        if input.fps:
+            files.setdefault("fps", input.fps)
 
-    if input.image_paths and len(input.image_paths) > 0:
-        image_path = input.image_paths[0]
-        image_file = open(image_path, "rb")
-        form.add_field(
-            "input_reference",
-            image_file,
-            filename=os.path.basename(image_path),
-            content_type=_guess_mime_type(image_path),
-        )
-    elif input.video_paths and len(input.video_paths) > 0:
-        video_path = input.video_paths[0]
-        if _is_minimax_h3_mixed_reference(input.model):
-            with open(video_path, "rb") as reference_file:
-                video_b64 = base64.b64encode(reference_file.read()).decode("ascii")
-            form.add_field(
-                "video_reference",
-                json.dumps([{"video_url": f"data:video/mp4;base64,{video_b64}"}]),
-            )
-        else:
-            image_file = open(video_path, "rb")
+        form = aiohttp.FormData()
+        for k, v in files.items():
+            form.add_field(k, str(v))
+
+        if input.image_paths and input.video_paths:
+            output.error = "Only one of image_paths or video_paths can be provided"
+            output.success = False
+            return output
+
+        if input.image_paths and len(input.image_paths) > 0:
+            image_path = input.image_paths[0]
+            image_file = open(image_path, "rb")
             form.add_field(
                 "input_reference",
                 image_file,
-                filename=os.path.basename(video_path),
-                content_type=_guess_mime_type(video_path),
+                filename=os.path.basename(image_path),
+                content_type=_guess_mime_type(image_path),
             )
+        elif input.video_paths and len(input.video_paths) > 0:
+            video_path = input.video_paths[0]
+            if _is_minimax_h3_mixed_reference(input.model):
+                with open(video_path, "rb") as reference_file:
+                    video_b64 = base64.b64encode(reference_file.read()).decode("ascii")
+                form.add_field(
+                    "video_reference",
+                    json.dumps([{"video_url": f"data:video/mp4;base64,{video_b64}"}]),
+                )
+            else:
+                image_file = open(video_path, "rb")
+                form.add_field(
+                    "input_reference",
+                    image_file,
+                    filename=os.path.basename(video_path),
+                    content_type=_guess_mime_type(video_path),
+                )
 
-    job_id = None
-    job_status = None
-    poll_json = {}
-    resp_json = {}
+        job_status = None
+        poll_json = {}
+        resp_json = {}
 
-    try:
         # invoke a post request (POST /v1/videos)
         async with session.post(input.api_url, data=form) as response:
             if response.status == 200:
@@ -406,7 +410,7 @@ async def async_request_v1_videos(
 
         # invoke a poll request (GET /v1/videos/{video_id})
         poll_interval = 2.0  # Unit(s)
-        timeout_seconds = 600.0
+        timeout_seconds = input.video_job_timeout
         deadline = time.perf_counter() + timeout_seconds
         job_url = f"{input.api_url}/{job_id}"
 
@@ -422,8 +426,11 @@ async def async_request_v1_videos(
                 poll_json = await poll_response.json()
                 job_status = poll_json.get("status")
 
-                if time.perf_counter() >= deadline:
-                    output.error = f"Timed out waiting for video job {job_id} to complete."
+                if job_status not in {"completed", "failed"} and time.perf_counter() >= deadline:
+                    output.error = (
+                        f"Timed out after {timeout_seconds:g}s waiting for video job {job_id} to complete. "
+                        "Increase --video-job-timeout for long-running or queued video jobs."
+                    )
                     output.success = False
                     return output
 
@@ -452,7 +459,7 @@ async def async_request_v1_videos(
             elif "peak_memory_mb" in resp_json:
                 output.peak_memory_mb = resp_json["peak_memory_mb"]
     except Exception as e:
-        output.error = str(e)
+        output.error = f"{type(e).__name__}: {e}"
         output.success = False
     finally:
         if image_file is not None:
@@ -465,13 +472,13 @@ async def async_request_v1_videos(
             except Exception as e:
                 print(f"Failed to clean up video job {job_id}: {e}")
 
-    output.latency = time.perf_counter() - output.start_time
+        output.latency = time.perf_counter() - output.start_time
+        if pbar is not None:
+            pbar.update(1)
 
     if output.success and input.slo_ms is not None:
         output.slo_achieved = (output.latency * 1000.0) <= float(input.slo_ms)
 
-    if pbar:
-        pbar.update(1)
     return output
 
 

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -1,6 +1,6 @@
 # adapted from fastvideo
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
 """
 Benchmark online serving for diffusion models (Image/Video Generation).
@@ -87,6 +87,7 @@ import base64
 import glob
 import json
 import logging
+import math
 import os
 import random
 import tempfile
@@ -1116,6 +1117,11 @@ def calculate_metrics(
         "duration": total_duration,
         "completed_requests": num_success,
         "failed_requests": len(error_outputs),
+        "request_errors": [
+            {"request_id": req.request_id, "error": out.error}
+            for req, out in zip(requests_list, outputs)
+            if not out.success
+        ],
         "throughput_qps": num_success / total_duration if total_duration > 0 else 0,
         "latency_mean": np.mean(latencies) if latencies else 0,
         "latency_median": np.median(latencies) if latencies else 0,
@@ -1261,6 +1267,9 @@ def _default_endpoint_for_task(task: str) -> str:
 
 
 async def benchmark(args):
+    if not math.isfinite(args.video_job_timeout) or args.video_job_timeout <= 0:
+        raise ValueError("--video-job-timeout must be a positive finite number of seconds.")
+
     # Construct base_url if not provided
     if args.base_url is None:
         args.base_url = f"http://{args.host}:{args.port}"
@@ -1312,6 +1321,8 @@ async def benchmark(args):
     print("Loading requests...")
     requests_list = dataset.get_requests()
     print(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.")
+    for req in requests_list:
+        req.video_job_timeout = args.video_job_timeout
 
     if args.return_stage_metrics and args.endpoint in _STAGE_METRICS_ENDPOINTS:
         for req in requests_list:
@@ -1376,6 +1387,8 @@ async def benchmark(args):
     metrics["model"] = args.model
     metrics["dataset"] = args.dataset
     metrics["task"] = args.task
+    if args.endpoint == "/v1/videos":
+        metrics["video_job_timeout"] = args.video_job_timeout
     if args.endpoint == "/v1/images/edits":
         metrics["bot_task"] = args.bot_task
 
@@ -1398,6 +1411,11 @@ async def benchmark(args):
         )
     )
     print("{:<40} {}/{:<15}".format("Successful requests:", metrics["completed_requests"], len(requests_list)))
+    print("{:<40} {:<15}".format("Failed requests:", metrics["failed_requests"]))
+    for failure in metrics["request_errors"][:10]:
+        print(f"Request {failure['request_id']} failed: {failure['error']}")
+    if metrics["failed_requests"] > 10:
+        print("Showing first 10 failures; use --output-file to save all request errors.")
 
     # Section 3: Performance Metrics
     print(f"{'-' * 50}")
@@ -1481,6 +1499,13 @@ if __name__ == "__main__":
         help="Path to local dataset file (optional).",
     )
     parser.add_argument("--num-prompts", type=int, default=10, help="Number of prompts to benchmark.")
+    parser.add_argument(
+        "--video-job-timeout",
+        type=float,
+        default=900.0,
+        help="Maximum polling time in seconds per video job, including server queue time (default: 900). "
+        "Increase this for long-running video generation or high concurrency.",
+    )
     parser.add_argument(
         "--max-concurrency",
         type=int,

benchmarks/edge_harness/PROFILING.md

@@ -0,0 +1,167 @@
+# Local model qualification and profiling
+
+These runners exercise existing Omni implementations with real weights. They
+do not add device or model support. A completed timing run is distinct from
+reference-quality, streaming, fault-recovery and release qualification.
+
+## Matrix and prerequisites
+
+The [60-cell audit](results/model_device_matrix_20260922/README.md) covers twelve
+PC/mobile/embedded configurations and five model families. Its raw evidence
+distinguishes complete pipelines from components and backend-only probes.
+Unavailable complete pipelines retain a reason and next repair step; a missing
+measurement is never represented as zero latency.
+
+Use the actual installed runtime and exact checkpoint identified in the result,
+including quantization, rather than substituting another family member. The
+current Spark CPU runner uses 1.7B INT8 and the CUDA runner uses 4B BF16. Their
+results do not establish same-model acceleration. Qwen3-TTS uses the pinned
+0.6B CustomVoice checkpoint, English/Vivian, through both Omni stages.
+
+Windows and WSL are distinct configurations on the same physical laptop.
+Run benchmarks serially: they share CPU, physical RAM, GPU and power limits.
+GPU board power is not whole-device energy. Windows used RAM includes WSL;
+do not add Windows and WSL counters as independent memory pools.
+
+## Run the workload
+
+Run from the repository root using the model's validated Python environment:
+
+```bash
+PYTHONPATH=. HF_HUB_OFFLINE=1 VLLM_USE_FLASHINFER_SAMPLER=0 \
+  python benchmarks/edge_harness/profile_local_text.py \
+  --model /absolute/path/to/checkpoint --out /absolute/path/to/new-run
+
+PYTHONPATH=. HF_HUB_OFFLINE=1 VLLM_USE_FLASHINFER_SAMPLER=0 \
+  python benchmarks/edge_harness/profile_local_tts.py \
+  --model /absolute/path/to/customvoice-checkpoint --out /absolute/path/to/new-run
+```
+
+Set the validated CPU thread/KV-cache settings or CUDA compiler environment
+before launching. Native Windows requires UTF-8 Python mode before interpreter
+startup (`PYTHONUTF8=1`, `PYTHONIOENCODING=utf-8`). The common benchmark helper
+uses the active environment's `torch/lib/cudart64_13.dll` when present and a
+short `~/c29` vLLM cache path unless explicitly overridden. These reproduce
+settings required by the tested Windows installation; they are not a general
+claim about other CUDA/runtime versions.
+
+Both runners require a new output directory and default to:
+
+- Three input lengths × concurrency 1/2/4 × 20 measured requests per group.
+- Separate warmup requests, excluded from group percentiles.
+- At least 1800 seconds of continuous medium-input requests.
+- Slow-consumer, cancellation and subsequent-request checks.
+
+Text timing forces 128 output tokens with greedy sampling and records actual
+input lengths, token IDs, sequence/epoch metadata and queue high-water marks.
+It does not recommend those sampling settings for applications. TTS requires
+DELTA output, records every audio chunk including terminal audio, verifies
+finite PCM, and archives warmup float32 chunks with hashes. Playback underruns
+are simulated from chunk arrivals; no sound-device playback or perceptual
+quality is implied.
+
+`profile_host_command.py` wraps an explicit command on Windows to collect
+one-second host-memory, AC/battery and whole-NVIDIA-device telemetry. Each host
+directory retains the exact command, exit status and combined log. Existing
+disk/JIT caches are used. Constructor time is not a cold-disk measurement and
+must remain separate from request latency. Sampled memory peaks are lower
+bounds, not proof that every transient allocation was observed.
+
+On Windows, a live reader can deny replacement of a report snapshot. The
+writer defers such running updates, records the permission error, and requires
+the final snapshot to persist. Request JSONL is the primary sample record.
+Read live reports through Linux when they reside in WSL, or use a Windows
+reader that permits file deletion/replacement. Failed profile/trace reports
+produce a nonzero exit code in the current runners; historical attempts may
+require inspection of the report rather than relying on the process exit code.
+
+## Separate diagnostics
+
+After timing runs finish, use `profile_trace_diagnostic.py --kind spark|tts`
+with the same `--model` and a fresh `--out`. It invokes Omni's profiler RPCs and
+measures three requests before, during and after tracing. Current text traces
+use 32 generated tokens and disable expensive shape/memory instrumentation.
+Compare paired diagnostic requests, not these 32-token times against the
+128-token baseline. Profiler start/stop/export costs are recorded separately
+when supported. The initial archived CPU diagnostic used full instrumentation
+and 128 tokens; its configuration and limitations are retained in its report.
+
+Run `check_text_reliability.py` or `check_tts_reliability.py` separately. They
+test real-model interruption/recovery and deliberately terminate a worker
+belonging to their own engine after verifying process ancestry. Text also
+checks actual consumer-queue saturation, stale handles and oversized-plan
+rejection. TTS's slow consumer does not prove saturation of every internal
+buffer. Inspect individual outcomes: `status=completed` means the probe ran,
+not that all checks passed. Any forced cleanup remains a finding.
+
+## Analyze and qualify
+
+For a separate TTS stage-attribution diagnostic, enable the existing
+`VLLM_OMNI_STEP_STATS_DIR=/absolute/path/to/counters` hook and use
+`VLLM_OMNI_STEP_STATS_SYNC=0`. Run `profile_local_tts.py` with
+`--length-band medium --concurrency 1 --repeats 20 --sustained-seconds 0`
+and a fresh output directory. This is not a full baseline run. The counters
+include warmup, slow-consumer, abort and recovery activity, and nested timers
+overlap. Preserve per-process summaries rather than pooling their percentiles.
+These are host-side timers without accelerator synchronization. Periodic
+flushing can omit the final counter tail after forced process termination.
+Comparisons with an uninstrumented baseline must retain cache/thermal and
+stochastic speech-length caveats.
+Core-step counts include idle/no-work iterations. The current orchestrator
+dispatch hook is only in the optional event-driven loop; the default loop
+does not produce that counter. Absent counters are not zero-time measurements.
+
+```bash
+python benchmarks/edge_harness/summarize_e2e_profiles.py \
+  --matrix benchmarks/edge_harness/results/model_device_matrix_20260922/matrix.json \
+  --runs /absolute/path/to/run-root --out /absolute/path/to/summary
+
+python benchmarks/edge_harness/analyze_profile_resources.py \
+  --runs /absolute/path/to/run-root --out /absolute/path/to/resource-summary.json
+```
+
+Use run directory names `spark-cpu-wsl`, `spark-cuda-wsl`,
+`spark-cuda-windows`, `tts-cuda-wsl` and `tts-cuda-windows`, with `-host`,
+`-trace` and `-reliability` suffixes for their corresponding records. The matrix
+summarizer verifies all sixty pairings and the original evidence hashes. It
+reports nearest-rank p50/p95 and sample counts, checks per-group coverage,
+flags recorded output-contract violations, and retains unavailable metrics.
+Batch-dependent greedy outputs are a numerical/reproducibility finding;
+timings alone cannot establish whether the cause is numerical or state-related.
+Delivery-spacing statistics pool adjacent intervals within each request; their
+sample count is the number of intervals. Text output updates may coalesce
+tokens, so this is delivery timing rather than isolated kernel inter-token
+latency. Effective prefill throughput includes all work before the first output.
+
+Parse large traces only after inference timing has finished. The resource
+analyzer streams trace JSON, separates summed event duration from interval
+union, and excludes telemetry gaps over five seconds from energy integration.
+Actual device placement, fallback, transfer and synchronization conclusions
+must be supported by the trace/runtime records, not merely the intended plan.
+The TTS diagnostic starts each stage with a distinct profiler prefix because
+both workers have rank 0 and the current runtime otherwise overwrites their
+shared trace filename. Both stages are active during traced requests. Sequential
+stop/export RPCs can extend later-stage trace windows with idle/export time;
+trace-window length is not model execution time or GPU utilization.
+Startup summaries retain both constructor time and inclusive host-launch to
+first warmup submission/output. The latter includes artifact hashing, plan
+construction and imports; constructor timing alone excludes that preparation.
+
+Keep compile/load/run failures and their exact configurations. Before declaring
+a pairing qualified, review model-specific reference quality, complete declared
+modalities, state/stream correctness, memory admission, executed placement and
+the performance protocol. Missing artifacts, backends, device access, sensors
+or reference suites require explicit reasons and repair actions.
+
+After every writer exits, archive evidence with reversible compression of large
+uncompressed files and an original/stored byte-hash manifest:
+
+```bash
+python benchmarks/edge_harness/archive_profiles.py create /path/to/run-root /path/to/new-archive
+python benchmarks/edge_harness/archive_profiles.py verify /path/to/new-archive
+python benchmarks/edge_harness/archive_profiles.py restore /path/to/new-archive /path/to/new-run-root
+```
+
+Restore before using original report-relative paths for compressed artifacts.
+The utility does not rewrite raw reports. Integrity verification establishes
+that archived bytes match, not that a model passes its qualification gates.

benchmarks/edge_harness/analyze_profile_resources.py

@@ -0,0 +1,299 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Summarize raw host telemetry and traces without adding overlapping timings."""
+
+import argparse
+import gzip
+import json
+import math
+import re
+from collections import defaultdict
+from pathlib import Path
+
+from summarize_e2e_profiles import read_lines, stats
+
+
+def interval_union(intervals):
+    total = 0.0
+    end = None
+    for start, stop in sorted(intervals):
+        if stop <= start:
+            continue
+        total += stop - start if end is None or start >= end else max(0, stop - end)
+        end = stop if end is None else max(end, stop)
+    return total
+
+
+def trace_events(path):
+    """Read one event at a time, avoiding a multi-GB decoded trace document."""
+    opener = gzip.open if path.suffix == ".gz" else open
+    with opener(path, "rt", encoding="utf-8") as stream:
+        buffer = ""
+        while True:
+            block = stream.read(65536)
+            if not block:
+                raise ValueError("No traceEvents array")
+            buffer += block
+            match = re.search(r'"traceEvents"\s*:\s*\[', buffer)
+            if match:
+                buffer = buffer[match.end() :]
+                break
+            if len(buffer) > 8 * 1024 * 1024:
+                raise ValueError("Trace header exceeds 8 MiB")
+        decoder = json.JSONDecoder()
+        while True:
+            buffer = buffer.lstrip(" \t\r\n,")
+            if buffer.startswith("]"):
+                return
+            try:
+                event, end = decoder.raw_decode(buffer)
+            except json.JSONDecodeError:
+                block = stream.read(65536)
+                if not block:
+                    raise ValueError("Truncated traceEvents array") from None
+                buffer += block
+                if len(buffer) > 16 * 1024 * 1024:
+                    raise ValueError("One trace event exceeds 16 MiB") from None
+                continue
+            buffer = buffer[end:]
+            yield event
+
+
+def trace_summary(path):
+    categories, ops = defaultdict(list), defaultdict(list)
+    for event in trace_events(path):
+        if event.get("ph") != "X" or not isinstance(event.get("dur"), (int, float)):
+            continue
+        categories[event.get("cat", "uncategorized")].append((event["ts"], event["ts"] + event["dur"]))
+        ops[(event.get("cat", ""), event.get("name", ""))].append(float(event["dur"]))
+    return {
+        "source": str(path),
+        "scope": "one process trace; event duration sums overlap and are not E2E time",
+        "categories": {
+            name: {
+                "events": len(group),
+                "duration_sum_us": sum(b - a for a, b in group),
+                "busy_union_us": interval_union(group),
+            }
+            for name, group in categories.items()
+        },
+        "largest_ops_by_duration_sum": sorted(
+            [
+                {"category": cat, "name": name, "duration_sum_us": sum(values), "duration_us": stats(values)}
+                for (cat, name), values in ops.items()
+            ],
+            key=lambda r: r["duration_sum_us"],
+            reverse=True,
+        )[:40],
+    }
+
+
+def telemetry_summary(path, start=None, stop=None):
+    rows = [
+        r for r in read_lines(path) if (start is None or r["unix"] >= start) and (stop is None or r["unix"] <= stop)
+    ]
+    series = defaultdict(list)
+    power = []
+    for row in rows:
+        series["host_available_bytes"].append(row["host_memory"]["available"])
+        series["host_used_bytes"].append(row["host_memory"]["used"])
+        series["host_cpu_percent"].append(row.get("host_cpu_percent"))
+        fields = row.get("gpu_csv", "").split(",")
+        if len(fields) != 10:
+            continue
+        for index, name in [
+            (2, "gpu_memory_mib"),
+            (4, "gpu_temperature_c"),
+            (5, "gpu_power_w"),
+            (6, "gpu_sm_clock_mhz"),
+            (7, "gpu_mem_clock_mhz"),
+            (8, "gpu_utilization_percent"),
+        ]:
+            try:
+                value = float(fields[index])
+            except ValueError:
+                continue
+            if not math.isfinite(value):
+                continue
+            series[name].append(value)
+            if index == 5:
+                power.append((row["unix"], value))
+    gaps = [b["unix"] - a["unix"] for a, b in zip(rows, rows[1:])]
+    intervals = [(a, b) for a, b in zip(power, power[1:]) if 0 < b[0] - a[0] <= 5]
+    energy = sum((b[0] - a[0]) * (a[1] + b[1]) / 2 for a, b in intervals)
+    return {
+        "source": str(path),
+        "samples": len(rows),
+        "span_s": rows[-1]["unix"] - rows[0]["unix"] if len(rows) >= 2 else None,
+        "requested_window": {"start": start, "stop": stop},
+        "sampling_gap_s": stats(gaps),
+        "gaps_over_5s": sum(g > 5 for g in gaps),
+        "metrics": {key: stats(values) for key, values in series.items()},
+        "ac_states": sorted({r["power"]["ACLineStatus"] for r in rows if "power" in r}),
+        "whole_gpu_energy_j_over_observed_intervals": energy if intervals else None,
+        "energy_observed_duration_s": sum(b[0] - a[0] for a, b in intervals),
+        "energy_scope": (
+            "Whole NVIDIA GPU incl. other processes; trapezoids skip gaps >5s; not whole-system, CPU or NPU energy."
+        ),
+    }
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--runs", type=Path, required=True)
+    p.add_argument("--out", type=Path, required=True)
+    p.add_argument(
+        "--skip-traces", action="store_true", help="Summarize telemetry only while inference timing is still running."
+    )
+    a = p.parse_args()
+    result = {"telemetry": {}, "traces": [], "trace_analysis": "deferred" if a.skip_traces else "requested"}
+    for host in sorted(a.runs.glob("*-host")):
+        raw = host / "telemetry.jsonl"
+        if not raw.is_file():
+            continue
+        name = host.name.removesuffix("-host")
+        record = {"whole_run": telemetry_summary(raw)}
+        report_path = a.runs / name / "report.json"
+        if report_path.is_file():
+            report = json.loads(report_path.read_text(encoding="utf-8"))
+            host_report_path = host / "status.json"
+            host_report = json.loads(host_report_path.read_text(encoding="utf-8")) if host_report_path.is_file() else {}
+            rows = read_lines(a.runs / name / "requests.jsonl")
+            warmups = [r for r in rows if r.get("phase") == "warmup"]
+            launch, report_start = host_report.get("start_unix"), report.get("start_unix")
+            log_path = host / "command.log"
+            timing_lines = []
+            log_lines = []
+            if log_path.is_file():
+                log_lines = log_path.read_text(encoding="utf-8", errors="replace").splitlines()
+                timing_lines = [
+                    line
+                    for line in log_lines
+                    if any(
+                        marker in line
+                        for marker in (
+                            "Loading model weights took",
+                            "Model loading took",
+                            "init engine (",
+                            "torch.compile took",
+                            "Captured talker_mtp graphs",
+                            "engine startup completed",
+                        )
+                    )
+                ]
+            record["lifecycle_log_observations"] = {
+                "scope": "Counts are log lines, not request counts or a correctness verdict. Inspect source logs.",
+                "markers": {},
+            }
+            for marker in ("Dropping output for unknown req", "force killing remaining process "):
+                matches = [line for line in log_lines if marker in line]
+                record["lifecycle_log_observations"]["markers"][marker] = {
+                    "count": len(matches),
+                    "examples": matches[:3] + matches[-3:] if len(matches) > 6 else matches,
+                }
+            first_submit = warmups[0].get("submitted_unix") if warmups else None
+            first_latency = warmups[0].get("ttft_s") if warmups else None
+            if first_latency is None and warmups and warmups[0].get("ttfa_ms") is not None:
+                first_latency = warmups[0]["ttfa_ms"] / 1000
+            record["startup"] = {
+                "cache_condition": report.get("cache_condition", "See source report"),
+                "constructor_s": report.get("startup_s"),
+                "host_launch_to_benchmark_report_start_s": report_start - launch if launch and report_start else None,
+                "host_launch_to_first_request_s": first_submit - launch if first_submit and launch else None,
+                "host_launch_to_first_output_s": (
+                    first_submit - launch + first_latency
+                    if first_submit and launch and first_latency is not None
+                    else None
+                ),
+                "inclusive_startup_scope": (
+                    "Host launch through first warmup submission/output includes imports, artifact hashing, "
+                    "plan construction and engine startup; uses recorded host/engine wall timestamps. "
+                    "Constructor timing alone excludes preparation before the constructor."
+                ),
+                "pre_constructor_scope": (
+                    "Host launch to report creation includes Python/import/preparation work; "
+                    "not isolated OS process creation."
+                ),
+                "first_warmup": {
+                    key: warmups[0].get(key)
+                    for key in (
+                        "request_id",
+                        "length_band",
+                        "concurrency",
+                        "wall_s",
+                        "ttft_s",
+                        "total_wall_s",
+                        "ttfa_ms",
+                        "rtf_total",
+                    )
+                }
+                if warmups
+                else None,
+                "runtime_loading_compilation_evidence": timing_lines,
+                "scope": (
+                    "Runtime-reported component lines retain original boundaries; "
+                    "do not sum parallel-stage times into startup latency."
+                ),
+            }
+            start, duration = report.get("sustained_start_unix"), report.get("sustained_wall_s")
+            if start and duration:
+                record["sustained"] = telemetry_summary(raw, start, start + duration)
+                record["sustained_first_5min"] = telemetry_summary(raw, start, start + 300)
+                record["sustained_last_5min"] = telemetry_summary(raw, start + duration - 300, start + duration)
+                record["sustained_last_60s"] = telemetry_summary(raw, start + duration - 60, start + duration)
+                requests = [r for r in rows if r.get("phase") == "sustained"]
+                record["sustained_request_summary"] = {
+                    "n": len(requests),
+                    "rtf_above_one_requests": sum((r.get("rtf_total") or 0) > 1 for r in requests),
+                    "simulated_playback_deficit_requests": sum((r.get("stall_at_ttfa_ms") or 0) > 0 for r in requests),
+                    "rtf_total": stats([r.get("rtf_total") for r in requests]),
+                    "scope": "All sustained requests; timing/real-time gates are separate from successful completion.",
+                }
+                record["sustained_request_trends"] = {}
+                for label, begin, end in [
+                    ("first_5min", start, start + 300),
+                    ("last_5min", start + duration - 300, start + duration),
+                    ("last_60s", start + duration - 60, start + duration),
+                ]:
+                    window = [r for r in requests if begin <= r.get("submitted_unix", -1) < end]
+                    record["sustained_request_trends"][label] = {
+                        "n": len(window),
+                        "window_basis": "request submission time",
+                        "metrics": {
+                            metric: stats([r.get(metric) for r in window])
+                            for metric in (
+                                "wall_s",
+                                "ttft_s",
+                                "decode_tok_per_s",
+                                "total_wall_s",
+                                "ttfa_ms",
+                                "rtf_total",
+                                "stall_at_ttfa_ms",
+                            )
+                        },
+                    }
+        result["telemetry"][name] = record
+    for path in [] if a.skip_traces else sorted(a.runs.glob("*trace*/traces/**/*")):
+        if path.is_file() and (path.name.endswith(".json") or path.name.endswith(".json.gz")):
+            report_path = a.runs / path.relative_to(a.runs).parts[0] / "report.json"
+            if (
+                not report_path.is_file()
+                or json.loads(report_path.read_text(encoding="utf-8")).get("status") != "completed"
+            ):
+                result["traces"].append(
+                    {
+                        "source": str(path),
+                        "status": "not_analyzed",
+                        "reason": "Diagnostic has no completed report; trace may be incomplete.",
+                    }
+                )
+                continue
+            try:
+                result["traces"].append(trace_summary(path))
+            except Exception as error:
+                result["traces"].append({"source": str(path), "error": repr(error)})
+    a.out.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
+    print(f"Summarized {len(result['telemetry'])} telemetry runs and {len(result['traces'])} traces")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/android_agent/README.md

@@ -0,0 +1,92 @@
+# Android agent toolkit for Spark-X2.5
+
+Lets a text-only model operate a phone: observe the accessibility tree, call
+one tool, execute it on the device, observe again.
+
+    ui_tree.py    uiautomator XML -> indexed element list -> back to a tap point
+    device.py     AdbDevice (a real phone) | SimulatedDevice (scripted, in-process)
+    executor.py   one tool call -> one device action, errors as sentences
+    tool_parse.py recover a call from whatever shape the model emitted
+    agent.py      the loop: budget, no-op detection, undo detection, fallbacks
+    run_episode.py CLI
+
+## Running it
+
+Serve the model (Spark emits GLM-4.5 tool format, so the stock parsers apply):
+
+    vllm serve <model> --port 8077 --enable-auto-tool-choice \
+        --tool-call-parser glm45 --reasoning-parser glm45
+
+Then:
+
+    python run_episode.py --task "Turn on Wi-Fi." --device sim   # no phone needed
+    python run_episode.py --task "Turn on Wi-Fi." --device adb   # a real phone
+
+`pytest test_android_agent.py` runs 21 tests with no phone and no server.
+
+## What the loop does that a plain ReAct loop does not
+
+Each one is a response to a measured failure of *this* model, not generic
+scaffolding:
+
+- **Retry without thinking on an empty reply.** The model can reason past its
+  token budget without closing `<think>`; vLLM's reasoning parser only emits
+  content once the block closes, so the reply arrives empty with no tool call.
+  Raising the cap buys a longer silence. A non-thinking retry costs ~0.6 s.
+- **Report actions that changed nothing.** "the screen did not change" is
+  information the model cannot otherwise have.
+- **Report and stop on self-undo.** Tapping a switch that is already on undoes
+  the task. Each tap really does change the screen, so a no-change check
+  cannot see it; the loop watches for the *same* action returning the screen to
+  its previous state. Deliberately narrow -- opening a submenu and pressing
+  back revisits a screen and must not trip it.
+- **Recover malformed tool calls.** See below.
+- **A step budget**, so a confused episode ends instead of running forever.
+
+The loop never decides a task is finished. It reports what happened and lets
+the model call `task_complete`; a harness that completes tasks on the model's
+behalf measures itself.
+
+## Measured: which build can actually finish a task
+
+Simulated phone, one run per cell, `max_steps` 10, Xeon 8480C 24 threads.
+
+| build | decode | "Turn on Wi-Fi." | "Set an alarm for 7:30 AM." |
+|---|---|---|---|
+| bf16 | 41.0 tok/s | done in 4 steps | wrong app (Settings, not Clock), toggles Airplane mode, stuck |
+| **W4A16 g128** | 80.0 tok/s | done in 4 steps | right app, full navigation (Clock -> Alarm -> Add -> Save), then no tool call |
+| W4A8 g128 | 86.6 tok/s | turns Wi-Fi on, then toggles it off again (2 runs, same failure) | taps "Phone" twice, reaches Alarm, then loops |
+
+Two things fall out, and both matter more than the throughput numbers:
+
+**Navigation survives quantization; completion does not.** Every build walks
+home -> Settings -> Network -> Wi-Fi correctly. Only some of them notice that
+the switch now reads `checked=true` and stop. W4A8 -- the fastest build, and
+the one the speed-first selector recommends -- turns Wi-Fi on and then taps it
+off again, ending with the task undone.
+
+**The fidelity number had a task-level meaning all along.** W4A8 reproduced
+1/12 greedy prompts against its own bf16 and W4A16 2/12
+(analysis/spark_edge_support.md 2c). That looked like a small difference on a
+proxy metric. It is the difference between finishing and not.
+
+So for agentic use the recommendation is **W4A16**, not W4A8: 8% slower, and
+it is the only 4-bit build here that completed anything. `weight_path.py`'s
+`priority="fidelity"` selects it.
+
+## Honest limits
+
+- **No phone was involved.** This host has no `adb`, no emulator and no
+  `/dev/kvm`. The simulator emits real uiautomator XML, so the parser, the
+  index-to-coordinate mapping and the stopping rules are exercised on the path
+  hardware drives -- but whether a real Settings app is laid out the way these
+  fixtures assume is untested.
+- **Two tasks, one run each.** Enough to show a failure mode reproduces, not
+  enough to rank builds by success rate.
+- **Stock `glm45` does not always parse Spark.** On 4 of 5 screens it does. On
+  the Wi-Fi toggle screen the model emitted
+  `[tap]\n{"tool": "tap", "parameters": {"index": 1}` -- unterminated, and
+  `tool_calls` came back empty. `tool_parse.py` recovers that and reports every
+  recovery, because an agent that silently repairs its model's output hides
+  regressions in that model. The earlier claim that the stock parsers work
+  "as-is" is true most of the time, which is not the same thing.

benchmarks/edge_harness/android_agent/action_space.py

@@ -0,0 +1,152 @@
+"""Android control action space for a text-only LLM agent.
+
+The tool surface mirrors what the mainstream ADB-based harnesses expose
+(droidrun/mobilerun's Portal tools, ghost-in-the-droid's touch/see tools,
+AndroidWorld's `JSONAction`), so an agent written against these schemas can be
+pointed at any of them. Every action is expressed over the accessibility tree
+rather than pixel coordinates, because Spark-X2.5 is a text-only model: it
+never sees a screenshot, it sees the uiautomator dump.
+
+`ADB_TEMPLATES` gives the literal `adb` invocation each tool maps to, which is
+what a driver executes against a real device.
+"""
+
+TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "tap",
+            "description": (
+                "Tap a UI element. Use the element's index exactly as shown in "
+                "the accessibility tree."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "index": {
+                        "type": "integer",
+                        "description": "Index of the element in the tree",
+                    }
+                },
+                "required": ["index"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "type_text",
+            "description": (
+                "Type text into a focused input field. Tap the field first."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "index": {"type": "integer", "description": "Index of the input field"},
+                    "text": {"type": "string", "description": "Text to enter"},
+                },
+                "required": ["index", "text"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "swipe",
+            "description": (
+                "Scroll the screen. The direction is where the content should "
+                "move: use 'down' to see what is further down the page."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "direction": {
+                        "type": "string",
+                        "enum": ["up", "down", "left", "right"],
+                    }
+                },
+                "required": ["direction"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "press_key",
+            "description": "Press a hardware or system key.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "key": {
+                        "type": "string",
+                        "enum": ["back", "home", "enter", "delete", "recent"],
+                    }
+                },
+                "required": ["key"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "open_app",
+            "description": "Launch an app by its visible name.",
+            "parameters": {
+                "type": "object",
+                "properties": {"name": {"type": "string"}},
+                "required": ["name"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "task_complete",
+            "description": (
+                "Call when the user's request has been fully carried out."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "summary": {"type": "string", "description": "What was done"}
+                },
+                "required": ["summary"],
+            },
+        },
+    },
+]
+
+# How each tool becomes a real device command. `{bounds}` is filled from the
+# element's centre point in the accessibility tree.
+ADB_TEMPLATES = {
+    "tap": "adb shell input tap {x} {y}",
+    "type_text": "adb shell input text {text_escaped}",
+    "swipe": "adb shell input swipe {x1} {y1} {x2} {y2} 300",
+    "press_key": "adb shell input keyevent {keycode}",
+    "open_app": "adb shell monkey -p {package} -c android.intent.category.LAUNCHER 1",
+}
+
+KEYCODES = {
+    "back": "KEYCODE_BACK",
+    "home": "KEYCODE_HOME",
+    "enter": "KEYCODE_ENTER",
+    "delete": "KEYCODE_DEL",
+    "recent": "KEYCODE_APP_SWITCH",
+}
+
+SYSTEM_PROMPT = (
+    "You control an Android phone. At each step you are given the current "
+    "screen as an accessibility tree: one line per element, with its index, "
+    "type, text and whether it is clickable. Call exactly one tool per step to "
+    "make progress on the user's task. Refer to elements only by the index "
+    "shown.\n"
+    # Both rules below exist because the model failed them in measurement: it
+    # tapped a Wi-Fi switch that already read checked=true, turning it back
+    # off, and never called task_complete. A switch's state is on the screen,
+    # so the rule can be stated in terms of what it can see.
+    "A switch or checkbox shows its state as checked=true or checked=false. "
+    "Tapping it flips that state.\n"
+    "Before acting, check whether the screen already shows what the task "
+    "asked for. If it does, call task_complete immediately and do not tap "
+    "anything else -- acting again would undo it."
+)

benchmarks/edge_harness/android_agent/agent.py

@@ -0,0 +1,271 @@
+"""The observe-decide-act loop.
+
+Three things in here are not generic agent scaffolding; they are direct
+responses to how this model failed when it was measured one step at a time
+(analysis/spark_edge_support.md section 5):
+
+* **A step budget, not a bigger token cap.** On the one genuinely ambiguous
+  screen the model reasoned past 1200 tokens without closing `<think>`, and
+  vLLM's reasoning parser only emits content once the block closes -- so the
+  reply came back empty, with no tool call at all. Raising the cap buys a
+  longer silence. The loop instead retries that step with thinking disabled,
+  which measured 0.61 s against 8.9 s, and takes the weaker-but-present answer.
+
+* **Detecting the no-op.** The clearest wrong action was tapping a Wi-Fi
+  switch that was already on -- undoing the task it had just completed. An
+  action that leaves the screen signature unchanged is reported back as such,
+  because "nothing happened" is information the model cannot otherwise have.
+
+* **Repeat suppression.** Two identical actions on an identical screen means
+  stuck, and a third will not help. The loop says so in the transcript and
+  stops, rather than burning the budget.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field
+from urllib import error as urlerror
+from urllib import request as urlrequest
+
+from action_space import SYSTEM_PROMPT, TOOLS
+from device import Device
+from executor import ActionResult, execute
+from tool_parse import extract_tool_call
+from ui_tree import Element, parse_ui_dump, render_tree, screen_signature
+
+
+@dataclass
+class AgentConfig:
+    base_url: str = "http://127.0.0.1:8077/v1"
+    model: str = "XHToken/Spark-X2.5-1.7B"
+    max_steps: int = 12
+    think: bool = True
+    think_fallback: bool = True
+    """On an empty reply, retry the step without thinking rather than fail."""
+    max_tokens: int = 1024
+    temperature: float = 0.0
+    timeout: float = 900.0
+    history: int = 6
+    retries: int = 3
+    """Transient HTTP failures retried before a step is abandoned."""
+    retry_backoff_s: float = 2.0
+    """Recent actions shown back to the model."""
+
+
+@dataclass
+class Step:
+    n: int
+    screen: str
+    tool: str | None
+    args: dict
+    result: ActionResult | None
+    latency_s: float
+    thinking: bool
+    note: str = ""
+    parsed_via: str = "tool_calls"
+    """Which parser recovered the call; anything but tool_calls is worth seeing."""
+
+
+@dataclass
+class Episode:
+    task: str
+    steps: list[Step] = field(default_factory=list)
+    done: bool = False
+    summary: str = ""
+    stop_reason: str = ""
+
+    @property
+    def n_steps(self) -> int:
+        return len(self.steps)
+
+    def transcript(self) -> str:
+        lines = [f"task: {self.task}"]
+        for s in self.steps:
+            call = f"{s.tool}({json.dumps(s.args, ensure_ascii=False)})" if s.tool else "(no tool call)"
+            outcome = s.result.message if s.result else s.note
+            if s.parsed_via not in ("tool_calls", "none"):
+                outcome += f"  [recovered via {s.parsed_via}]"
+            lines.append(f"  {s.n:2d}. {call:48s} -> {outcome}  [{s.latency_s:.2f}s]")
+        lines.append(f"stopped: {self.stop_reason}")
+        return "\n".join(lines)
+
+
+class ChatError(RuntimeError):
+    pass
+
+
+def chat(cfg: AgentConfig, messages: list[dict], *, think: bool) -> tuple[dict, float]:
+    payload = {
+        "model": cfg.model,
+        "messages": messages,
+        "tools": TOOLS,
+        "tool_choice": "auto",
+        "temperature": cfg.temperature,
+        "max_tokens": cfg.max_tokens,
+    }
+    if not think:
+        # Spark's chat template opens <think> by default; closing it up front
+        # is what makes the model answer directly.
+        payload["chat_template_kwargs"] = {"enable_thinking": False}
+    req = urlrequest.Request(
+        f"{cfg.base_url}/chat/completions",
+        data=json.dumps(payload).encode(),
+        headers={"Content-Type": "application/json"},
+    )
+    t0 = time.perf_counter()
+    last: Exception | None = None
+    for attempt in range(max(1, cfg.retries)):
+        try:
+            with urlrequest.urlopen(req, timeout=cfg.timeout) as r:
+                return json.loads(r.read()), time.perf_counter() - t0
+        except (urlerror.URLError, TimeoutError, json.JSONDecodeError) as exc:
+            # A phone task is a dozen round trips; one dropped connection
+            # should cost a retry, not the episode.
+            last = exc
+            if attempt + 1 < max(1, cfg.retries):
+                time.sleep(cfg.retry_backoff_s * (attempt + 1))
+    raise ChatError(
+        f"{cfg.base_url} failed after {cfg.retries} attempts: {last}") from last
+
+
+def wait_for_server(cfg: AgentConfig, timeout_s: float = 300.0) -> bool:
+    """Block until the endpoint serves a model list, or give up.
+
+    Starting an episode against a server that is still loading produces a
+    confusing transcript full of connection errors; this turns that into one
+    clear wait.
+    """
+    deadline = time.monotonic() + timeout_s
+    while time.monotonic() < deadline:
+        try:
+            with urlrequest.urlopen(f"{cfg.base_url}/models", timeout=5) as r:
+                if b'"id"' in r.read():
+                    return True
+        except Exception:
+            time.sleep(2.0)
+    return False
+
+
+def parse_response(response: dict):
+    """The one call to act on, from whichever shape the model used.
+
+    Spark does not always stay in GLM-4.5 tool format -- see tool_parse -- and
+    on a phone a dropped step is a stalled task, so the fallbacks matter.
+    """
+    try:
+        message = response["choices"][0]["message"]
+    except (KeyError, IndexError):
+        message = {}
+    return extract_tool_call(message)
+
+
+def first_tool_call(response: dict) -> tuple[str | None, dict]:
+    """Name and arguments only, for callers that do not care how it parsed."""
+    parsed = parse_response(response)
+    return parsed.name, parsed.args
+
+
+def _user_message(task: str, screen: str, log: list[str], history: int) -> str:
+    parts = [f"Task: {task}", "", "Current screen:", screen]
+    if log:
+        parts += ["", "Actions so far:"] + [f"  {line}" for line in log[-history:]]
+    parts += ["", "Choose the next action. Call exactly one tool."]
+    return "\n".join(parts)
+
+
+def observe(device: Device) -> tuple[list[Element], str, str]:
+    elements = parse_ui_dump(device.dump())
+    return elements, render_tree(elements), screen_signature(elements)
+
+
+def run_episode(device: Device, task: str, cfg: AgentConfig | None = None) -> Episode:
+    cfg = cfg or AgentConfig()
+    episode = Episode(task=task)
+    log: list[str] = []
+    recent: list[tuple[str, str]] = []  # (action, screen signature before it)
+    signature_before_last: str | None = None
+    last_action: str | None = None
+    oscillations = 0
+
+    for n in range(1, cfg.max_steps + 1):
+        elements, screen, signature = observe(device)
+        messages = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user",
+             "content": _user_message(task, screen, log, cfg.history)},
+        ]
+
+        thinking = cfg.think
+        response, latency = chat(cfg, messages, think=thinking)
+        parsed = parse_response(response)
+        tool, args = parsed.name, parsed.args
+
+        if tool is None and cfg.think_fallback and thinking:
+            # The measured failure: reasoning never closed, so the parser
+            # emitted nothing. A second, non-thinking pass is worth ~0.6 s.
+            thinking = False
+            retry, retry_latency = chat(cfg, messages, think=False)
+            latency += retry_latency
+            parsed = parse_response(retry)
+            tool, args = parsed.name, parsed.args
+
+        if tool is None:
+            episode.steps.append(Step(n, screen, None, {}, None, latency, thinking,
+                                      note="model returned no tool call",
+                                      parsed_via=parsed.source))
+            episode.stop_reason = "no tool call, even without thinking"
+            return episode
+
+        if (tool_key := f"{tool}:{json.dumps(args, sort_keys=True)}") and \
+                recent.count((tool_key, signature)) >= 2:
+            episode.steps.append(Step(n, screen, tool, args, None, latency, thinking,
+                                      note="repeated a no-op action",
+                                      parsed_via=parsed.source))
+            episode.stop_reason = (
+                f"stuck: {tool} repeated on an unchanged screen"
+            )
+            return episode
+        recent.append((tool_key, signature))
+
+        result = execute(tool, args, elements, device)
+        if result.ok and not result.done:
+            _, _, after = observe(device)
+            if after == signature:
+                # Say so out loud: the model has no other way to learn that
+                # its action changed nothing.
+                result = ActionResult(True, result.message + " -- the screen did not change",
+                                      action=result.action)
+            elif tool_key == last_action and after == signature_before_last:
+                # Oscillation: the *same* action, undoing what it just did --
+                # the measured failure, tapping a switch off and on forever.
+                # It hides from the no-change check because each individual tap
+                # really does change the screen. Deliberately narrow: revisiting
+                # a screen by other means (open a submenu, press back) is normal
+                # navigation and must not trip this.
+                result = ActionResult(
+                    True,
+                    result.message + " -- that undid your previous action; the "
+                    "setting is now back where it started",
+                    action=result.action,
+                )
+                oscillations += 1
+                if oscillations >= 2:
+                    episode.steps.append(Step(n, screen, tool, args, result, latency,
+                                              thinking, parsed_via=parsed.source))
+                    episode.stop_reason = "stuck: repeatedly undoing its own action"
+                    return episode
+        signature_before_last, last_action = signature, tool_key
+        episode.steps.append(Step(n, screen, tool, args, result, latency, thinking,
+                                  parsed_via=parsed.source))
+        log.append(f"{tool} -> {result.message}")
+
+        if result.done:
+            episode.done = True
+            episode.summary = result.summary
+            episode.stop_reason = "task_complete"
+            return episode
+
+    episode.stop_reason = f"step budget exhausted ({cfg.max_steps})"
+    return episode

benchmarks/edge_harness/android_agent/device.py

@@ -0,0 +1,354 @@
+"""Where the agent's actions actually land.
+
+Two backends behind one interface. `AdbDevice` talks to a phone over `adb`.
+`SimulatedDevice` runs a small scripted phone in-process, and exists for a
+reason beyond convenience: no Android device, emulator or `/dev/kvm` is
+reachable from the host this was built on, so without it the loop, the tool
+executor and the stopping rules would all be untested code shipped on a
+promise.
+
+The simulator emits **real uiautomator XML**, not pre-rendered text, so the
+parser, the index-to-coordinate mapping and the screen-change detection are
+exercised on the same path a phone would drive. What it cannot test is
+whether a real Settings app is laid out the way the fixtures assume; that is
+stated in the report rather than hidden.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from xml.sax.saxutils import escape as _xml_escape
+import time
+from dataclasses import dataclass, field
+from typing import Callable, Protocol
+
+KEYCODES = {
+    "back": "KEYCODE_BACK",
+    "home": "KEYCODE_HOME",
+    "enter": "KEYCODE_ENTER",
+    "delete": "KEYCODE_DEL",
+    "recent": "KEYCODE_APP_SWITCH",
+}
+
+
+class Device(Protocol):
+    def dump(self) -> str: ...
+    def tap(self, x: int, y: int) -> None: ...
+    def input_text(self, text: str) -> None: ...
+    def swipe(self, x1: int, y1: int, x2: int, y2: int, ms: int = 300) -> None: ...
+    def keyevent(self, key: str) -> None: ...
+    def launch(self, app: str) -> None: ...
+    def size(self) -> tuple[int, int]: ...
+
+
+class AdbError(RuntimeError):
+    pass
+
+
+class AdbDevice:
+    """A real phone over `adb`."""
+
+    def __init__(self, serial: str | None = None, adb: str = "adb",
+                 timeout: float = 30.0):
+        if shutil.which(adb) is None:
+            raise AdbError(
+                f"{adb!r} is not on PATH. Install platform-tools and connect a "
+                "device (`adb devices` should list it)."
+            )
+        self.adb, self.serial, self.timeout = adb, serial, timeout
+        self._size: tuple[int, int] | None = None
+
+    def _run(self, *args: str, binary: bool = False):
+        cmd = [self.adb] + (["-s", self.serial] if self.serial else []) + list(args)
+        proc = subprocess.run(cmd, capture_output=True, timeout=self.timeout)
+        if proc.returncode != 0:
+            raise AdbError(f"{' '.join(cmd)} failed: {proc.stderr.decode(errors='replace')[:400]}")
+        return proc.stdout if binary else proc.stdout.decode(errors="replace")
+
+    def dump(self) -> str:
+        # exec-out keeps the XML on stdout instead of round-tripping through
+        # /sdcard, which is both faster and avoids a writable-storage
+        # assumption that does not hold on every device.
+        out = self._run("exec-out", "uiautomator", "dump", "/dev/tty")
+        start = out.find("<?xml")
+        if start < 0:
+            start = out.find("<hierarchy")
+        if start < 0:
+            raise AdbError(f"no UI hierarchy in dump output: {out[:200]!r}")
+        end = out.rfind("</hierarchy>")
+        return out[start:end + len("</hierarchy>")] if end > 0 else out[start:]
+
+    def tap(self, x: int, y: int) -> None:
+        self._run("shell", "input", "tap", str(x), str(y))
+
+    def input_text(self, text: str) -> None:
+        # `input text` takes a single token: spaces become %s and the shell
+        # metacharacters have to go, or half the string is eaten.
+        escaped = text.replace(" ", "%s")
+        for ch in "()<>|;&*\\~\"'`$":
+            escaped = escaped.replace(ch, "\\" + ch)
+        self._run("shell", "input", "text", escaped)
+
+    def swipe(self, x1: int, y1: int, x2: int, y2: int, ms: int = 300) -> None:
+        self._run("shell", "input", "swipe", str(x1), str(y1), str(x2), str(y2), str(ms))
+
+    def keyevent(self, key: str) -> None:
+        code = KEYCODES.get(key, key)
+        self._run("shell", "input", "keyevent", code)
+
+    def launch(self, app: str) -> None:
+        """Launch by package name, or by visible label via the launcher."""
+        if "." in app:
+            self._run("shell", "monkey", "-p", app, "-c",
+                      "android.intent.category.LAUNCHER", "1")
+            return
+        raise AdbError(
+            f"open_app needs a package name, got {app!r}. Resolve the label to a "
+            "package first (`adb shell pm list packages`) or tap its icon."
+        )
+
+    def size(self) -> tuple[int, int]:
+        if self._size is None:
+            out = self._run("shell", "wm", "size")
+            part = out.strip().rsplit(":", 1)[-1].strip()
+            w, h = part.split("x")
+            self._size = (int(w), int(h))
+        return self._size
+
+
+# --------------------------------------------------------------------------
+# Simulated phone
+# --------------------------------------------------------------------------
+
+@dataclass
+class SimElement:
+    label: str
+    cls: str = "android.widget.TextView"
+    clickable: bool = True
+    checkable: bool = False
+    checked: bool = False
+    selected: bool = False
+    editable: bool = False
+    goto: str | None = None
+    """Screen this element navigates to when tapped."""
+    toggles: str | None = None
+    """State key this element flips when tapped."""
+    saves: str | None = None
+    """Record key. Tapping commits the current field values into
+    ``records[saves]`` and clears them -- the difference between typing a time
+    and having an alarm, which the scored suite previously could not tell."""
+    label_of: str | None = None
+    """State key whose value picks the label from ``labels``."""
+    labels: tuple[str, str] = ("", "")
+    """(label when the state key is false, label when true)."""
+
+    def rendered_label(self, state: dict) -> str:
+        if self.label_of:
+            return self.labels[bool(state.get(self.label_of))]
+        return self.label
+
+
+@dataclass
+class SimScreen:
+    screen_id: str
+    elements: list[SimElement]
+    back: str | None = None
+    dynamic: "Callable[[SimulatedDevice], list[SimElement]] | None" = None
+    """Rows derived from device state, appended after ``elements``. A saved
+    alarm has to appear *on the screen*, or an oracle that reads the screen --
+    which is the only kind a real phone can have -- cannot tell a committed
+    alarm from a typed one."""
+
+
+@dataclass
+class SimulatedDevice:
+    """A scripted phone that speaks uiautomator XML.
+
+    Deliberately small. It is here to prove the loop terminates, recovers and
+    stops -- not to model Android.
+    """
+
+    screens: dict[str, SimScreen]
+    current: str
+    state: dict[str, bool] = field(default_factory=dict)
+    width: int = 1080
+    height: int = 2400
+    typed: list[str] = field(default_factory=list)
+    actions: list[tuple[str, ...]] = field(default_factory=list)
+    home: str = "home"
+    focus: str | None = None
+    """Label of the focused text field; typing lands here."""
+    fields: dict[str, str] = field(default_factory=dict)
+    records: dict[str, list[dict]] = field(default_factory=dict)
+    toggled: list[tuple[str, bool, bool]] = field(default_factory=list)
+    """(key, before, after) per switch tap. A task whose starting state is
+    already correct is failed by a round trip, which final state cannot see."""
+
+    _ROW_H = 160
+    _TOP = 200
+
+    def _visible(self) -> list[SimElement]:
+        screen = self.screens[self.current]
+        rows = list(screen.elements)
+        if screen.dynamic:
+            rows = [r for r in rows if not getattr(r, "_placeholder", False)]
+            rows.extend(screen.dynamic(self))
+        return rows
+
+    def _bounds(self, row: int) -> tuple[int, int, int, int]:
+        y1 = self._TOP + row * self._ROW_H
+        return 0, y1, self.width, y1 + self._ROW_H
+
+    def dump(self) -> str:
+        rows = []
+        for i, el in enumerate(self._visible()):
+            x1, y1, x2, y2 = self._bounds(i)
+            checked = el.checked
+            if el.toggles:
+                checked = self.state.get(el.toggles, el.checked)
+            # uiautomator escapes attribute values, and labels really do
+            # contain them -- "Network & internet" is on every Android
+            # settings screen. Emitting raw breaks the parser on the second
+            # screen of the first task.
+            label = _xml_escape(el.rendered_label(self.state), {'"': "&quot;"})
+            rows.append(
+                f'  <node index="{i}" text="{label}" resource-id="" '
+                f'class="{el.cls}" package="com.example.sim" content-desc="" '
+                f'checkable="{str(el.checkable).lower()}" '
+                f'checked="{str(checked).lower()}" '
+                f'clickable="{str(el.clickable).lower()}" enabled="true" '
+                f'focusable="{str(el.clickable).lower()}" focused="false" '
+                f'scrollable="false" long-clickable="false" password="false" '
+                f'selected="{str(el.selected).lower()}" '
+                f'bounds="[{x1},{y1}][{x2},{y2}]" />'
+            )
+        body = "\n".join(rows)
+        return (
+            '<?xml version="1.0" encoding="UTF-8"?>\n'
+            f'<hierarchy rotation="0">\n{body}\n</hierarchy>'
+        )
+
+    def _row_at(self, y: int) -> int | None:
+        row = (y - self._TOP) // self._ROW_H
+        return row if 0 <= row < len(self._visible()) else None
+
+    def tap(self, x: int, y: int) -> None:
+        self.actions.append(("tap", str(x), str(y)))
+        row = self._row_at(y)
+        if row is None:
+            return
+        el = self._visible()[row]
+        if el.editable:
+            self.focus = el.label
+        if el.toggles:
+            before = bool(self.state.get(el.toggles, el.checked))
+            self.state[el.toggles] = not before
+            self.toggled.append((el.toggles, before, not before))
+        if el.saves:
+            self.records.setdefault(el.saves, []).append(
+                {**self.fields, **{k: v for k, v in self.state.items()
+                                   if k.startswith(el.saves + "_")}})
+            self.fields = {}
+            self.focus = None
+        if el.goto and el.goto in self.screens:
+            self.current = el.goto
+
+    def input_text(self, text: str) -> None:
+        self.actions.append(("text", text))
+        self.typed.append(text)
+        if self.focus is not None:
+            self.fields[self.focus] = self.fields.get(self.focus, "") + text
+
+    def swipe(self, x1: int, y1: int, x2: int, y2: int, ms: int = 300) -> None:
+        self.actions.append(("swipe", str(x1), str(y1), str(x2), str(y2)))
+
+    def keyevent(self, key: str) -> None:
+        self.actions.append(("key", key))
+        if key == "back":
+            self.current = self.screens[self.current].back or self.home
+        elif key == "home":
+            self.current = self.home
+
+    def launch(self, app: str) -> None:
+        self.actions.append(("launch", app))
+        for screen in self.screens.values():
+            for el in screen.elements:
+                if el.label.lower() == app.lower() and el.goto:
+                    self.current = el.goto
+                    return
+
+    def size(self) -> tuple[int, int]:
+        return self.width, self.height
+
+
+def _saved_alarm_rows(device: "SimulatedDevice") -> list[SimElement]:
+    """Render committed alarms the way a clock app lists them."""
+    saved = device.records.get("alarm", [])
+    if not saved:
+        return [SimElement("No alarms", clickable=False)]
+    rows = []
+    for rec in saved:
+        hour = (rec.get("Hour") or "?").lstrip("0") or "0"
+        minute = (rec.get("Minute") or "?").rjust(2, "0")
+        meridiem = "PM" if rec.get("alarm_pm") else "AM"
+        rows.append(SimElement(f"{hour}:{minute} {meridiem}", clickable=False))
+    return rows
+
+
+def demo_phone() -> SimulatedDevice:
+    """A two-app phone: Settings (Wi-Fi) and Clock (alarms).
+
+    Enough to exercise a multi-screen task, a stateful toggle -- the case the
+    earlier single-step run got wrong, tapping a switch that was already on --
+    and a text field.
+    """
+    screens = {
+        "home": SimScreen("home", [
+            SimElement("Phone"),
+            SimElement("Messages"),
+            SimElement("Clock", goto="clock"),
+            SimElement("Settings", goto="settings"),
+            SimElement("Camera"),
+        ]),
+        "settings": SimScreen("settings", [
+            SimElement("Settings", clickable=False),
+            SimElement("Network & internet", goto="network"),
+            SimElement("Connected devices"),
+            SimElement("Apps"),
+            SimElement("Battery"),
+        ], back="home"),
+        "network": SimScreen("network", [
+            SimElement("Network & internet", clickable=False),
+            SimElement("Wi-Fi", cls="android.widget.Switch", checkable=True,
+                       toggles="wifi"),
+            SimElement("Mobile network"),
+            SimElement("Airplane mode", cls="android.widget.Switch",
+                       checkable=True, toggles="airplane"),
+        ], back="settings"),
+        "clock": SimScreen("clock", [
+            SimElement("Alarm", cls="android.widget.Button", goto="alarms"),
+            SimElement("Clock", cls="android.widget.Button", selected=True),
+            SimElement("Timer", cls="android.widget.Button"),
+            SimElement("Stopwatch", cls="android.widget.Button"),
+        ], back="home"),
+        "alarms": SimScreen("alarms", [
+            SimElement("Alarm", cls="android.widget.Button", selected=True),
+            SimElement("Add alarm", cls="android.widget.ImageButton",
+                       goto="new_alarm"),
+        ], back="clock", dynamic=_saved_alarm_rows),
+        "new_alarm": SimScreen("new_alarm", [
+            SimElement("Set time", clickable=False),
+            SimElement("Hour", cls="android.widget.EditText", editable=True),
+            SimElement("Minute", cls="android.widget.EditText", editable=True),
+            # Android shows one AM/PM control whose label is its value, so
+            # "set an alarm for 7:30 AM" has a wrong answer as well as a right
+            # one. Without it the task could be passed by typing any digits.
+            SimElement("AM", cls="android.widget.Button",
+                       label_of="alarm_pm", labels=("AM", "PM"),
+                       toggles="alarm_pm"),
+            SimElement("Save", cls="android.widget.Button", goto="alarms",
+                       saves="alarm"),
+        ], back="alarms"),
+    }
+    return SimulatedDevice(screens=screens, current="home", state={"wifi": False})

benchmarks/edge_harness/android_agent/executor.py

@@ -0,0 +1,147 @@
+"""Turn one tool call into one device action.
+
+Every failure here is returned to the model as a sentence it can act on, not
+raised. A 1.7B model asked for `index: 9` on a four-element screen has made a
+recoverable mistake, and "index 9 is out of range, the screen has 4 elements
+(0-3)" is the difference between the next step being right and the episode
+ending. Exceptions are reserved for the device being gone.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from device import KEYCODES, Device
+from ui_tree import Element
+
+
+@dataclass
+class ActionResult:
+    ok: bool
+    message: str
+    done: bool = False
+    summary: str = ""
+    action: str = ""
+
+    def as_tool_reply(self) -> str:
+        return self.message
+
+
+def _need(args: dict, key: str, kind: type, what: str) -> tuple[object, str | None]:
+    if key not in args:
+        return None, f"missing required argument {key!r} for {what}"
+    value = args[key]
+    if kind is int:
+        try:
+            value = int(value)
+        except (TypeError, ValueError):
+            return None, f"{key!r} must be a whole number, got {args[key]!r}"
+    elif not isinstance(value, kind):
+        return None, f"{key!r} must be {kind.__name__}, got {type(value).__name__}"
+    return value, None
+
+
+def _resolve(index: object, elements: list[Element]) -> tuple[Element | None, str | None]:
+    i = int(index)
+    if not elements:
+        return None, "the screen has no addressable elements; try press_key(back)"
+    if not 0 <= i < len(elements):
+        return None, (
+            f"index {i} is out of range; the screen has {len(elements)} "
+            f"elements (0-{len(elements) - 1})"
+        )
+    return elements[i], None
+
+
+def execute(name: str, args: dict, elements: list[Element],
+            device: Device) -> ActionResult:
+    """Run one tool call. Returns what to tell the model."""
+    args = args or {}
+
+    if name == "task_complete":
+        summary = str(args.get("summary", "")).strip()
+        return ActionResult(True, "task marked complete", done=True,
+                            summary=summary, action="task_complete")
+
+    if name == "tap":
+        index, err = _need(args, "index", int, "tap")
+        if err:
+            return ActionResult(False, err, action="tap")
+        el, err = _resolve(index, elements)
+        if err:
+            return ActionResult(False, err, action="tap")
+        x, y = el.center
+        device.tap(x, y)
+        state = ""
+        if "checked" in el.flags:
+            state = f" (was {'on' if el.flags['checked'] else 'off'})"
+        return ActionResult(True, f'tapped [{index}] "{el.text}"{state}',
+                            action=f"tap:{index}")
+
+    if name == "type_text":
+        index, err = _need(args, "index", int, "type_text")
+        if err:
+            return ActionResult(False, err, action="type_text")
+        text, err = _need(args, "text", str, "type_text")
+        if err:
+            return ActionResult(False, err, action="type_text")
+        el, err = _resolve(index, elements)
+        if err:
+            return ActionResult(False, err, action="type_text")
+        # Focus first: `input text` goes to whatever holds focus, which on a
+        # freshly dumped screen is usually nothing.
+        x, y = el.center
+        device.tap(x, y)
+        device.input_text(str(text))
+        return ActionResult(True, f'typed "{text}" into [{index}] "{el.text}"',
+                            action=f"type:{index}")
+
+    if name == "swipe":
+        direction = str(args.get("direction", "")).lower()
+        if direction not in ("up", "down", "left", "right"):
+            return ActionResult(
+                False, "direction must be one of up, down, left, right",
+                action="swipe")
+        w, h = device.size()
+        cx, cy = w // 2, h // 2
+        dx, dy = int(w * 0.35), int(h * 0.30)
+        # `direction` is where the *content* should move, i.e. what the user
+        # means by "scroll down to see more", so the finger travels the other
+        # way. Stated in the tool description too, because guessing this wrong
+        # is invisible in the transcript.
+        moves = {
+            "down": (cx, cy + dy, cx, cy - dy),
+            "up": (cx, cy - dy, cx, cy + dy),
+            "right": (cx + dx, cy, cx - dx, cy),
+            "left": (cx - dx, cy, cx + dx, cy),
+        }
+        device.swipe(*moves[direction])
+        return ActionResult(True, f"swiped to scroll {direction}",
+                            action=f"swipe:{direction}")
+
+    if name == "press_key":
+        key = str(args.get("key", "")).lower()
+        if key not in KEYCODES:
+            return ActionResult(
+                False, f"unknown key {key!r}; use one of {', '.join(KEYCODES)}",
+                action="press_key")
+        device.keyevent(key)
+        return ActionResult(True, f"pressed {key}", action=f"key:{key}")
+
+    if name == "open_app":
+        app = str(args.get("name", "")).strip()
+        if not app:
+            return ActionResult(False, "open_app needs a name", action="open_app")
+        try:
+            device.launch(app)
+        except Exception as exc:  # device-specific launch failures are recoverable
+            return ActionResult(False, f"could not launch {app!r}: {exc}",
+                                action="open_app")
+        return ActionResult(True, f"launched {app}", action=f"open:{app}")
+
+    return ActionResult(
+        False,
+        f"unknown tool {name!r}; available: tap, type_text, swipe, press_key, "
+        "open_app, task_complete",
+        action="unknown",
+    )

benchmarks/edge_harness/android_agent/phone_state.py

@@ -0,0 +1,135 @@
+"""What the phone is actually in, read the same way for both backends.
+
+The scored suite used to take its predicates over ``SimulatedDevice``'s Python
+attributes, so every scored run was a simulator run by construction -- the
+real-device path existed for *driving* a phone and not for judging one. That
+made "the agent suite" a claim about fixtures.
+
+Success is therefore read two ways, and the order matters:
+
+1. **Out of band, from the setting itself** -- ``device.state`` in the
+   simulator, ``settings get global wifi_on`` over adb. This is the ground
+   truth, and it does not depend on where the agent happened to leave the
+   phone. Scoring from the final screen alone fails an agent that turns Wi-Fi
+   on correctly and then presses Home, which is not a failure.
+2. **From the UI dump**, for anything with no setting behind it -- a saved
+   alarm appears as a row in the clock app and nowhere else. The simulator
+   emits real uiautomator XML, so this goes down the same parse, the same
+   switch lookup and the same text matching a phone would.
+
+Only the facts a phone genuinely cannot report after the fact -- which
+switches were flipped on the way, and what was typed into which field -- come
+from the simulator object, and any task needing them declares it, so a
+real-device run refuses to score rather than scoring wrong.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from ui_tree import parse_ui_dump
+
+
+def _norm(s: str) -> str:
+    """Match labels across the punctuation Android is inconsistent about."""
+    return "".join(c for c in s.lower() if c.isalnum())
+
+
+@dataclass
+class PhoneState:
+    source: str
+    """``"sim"`` or ``"device"`` -- which facts below are trustworthy."""
+    settings: dict[str, bool] = field(default_factory=dict)
+    """Authoritative setting values, read out of band. Preferred over
+    :attr:`switches`, which only shows what is currently on screen."""
+    screen: str | None = None
+    """Simulator screen id. ``None`` on a real phone, where there is no such
+    thing; use :meth:`has_text` against something the screen actually shows."""
+    switches: dict[str, bool] = field(default_factory=dict)
+    texts: list[str] = field(default_factory=list)
+    typed: list[str] = field(default_factory=list)
+    fields: dict[str, str] = field(default_factory=dict)
+    records: dict[str, list[dict]] = field(default_factory=dict)
+    toggled: list[tuple[str, bool, bool]] = field(default_factory=list)
+    """(key, before, after) for every switch the agent actually flipped."""
+
+    def switch(self, *names: str) -> bool | None:
+        """State of the first control matching any of ``names``.
+
+        The setting wins over the screen: an agent that turned Wi-Fi on and
+        then navigated away has still turned Wi-Fi on. ``None`` means neither
+        source knows the control, which is not the same as off -- an agent that
+        never found the screen has not turned anything on, and predicates must
+        not read that as "off".
+        """
+        wanted = {_norm(n) for n in names}
+        for key, value in self.settings.items():
+            if _norm(key) in wanted:
+                return bool(value)
+        for label, checked in self.switches.items():
+            if _norm(label) in wanted:
+                return checked
+        return None
+
+    def has_text(self, *needles: str) -> bool:
+        joined = " ".join(self.texts).lower()
+        return any(n.lower() in joined for n in needles)
+
+    def times_toggled(self, key: str) -> int:
+        return sum(1 for k, _, _ in self.toggled if k == key)
+
+
+def read_state(device) -> PhoneState:
+    """Read the phone through its screen, plus the simulator's own records."""
+    elements = parse_ui_dump(device.dump())
+    # `checkable` is deliberately not among the flags ui_tree parses, because
+    # that list is also what the model is shown; adding to it would change the
+    # agent's prompt. A switch is recognised by its class or by exposing a
+    # `checked` flag instead.
+    switches = {e.text: bool(e.flags.get("checked", False))
+                for e in elements
+                if "checked" in e.flags
+                or e.short_class in ("Switch", "CheckBox", "ToggleButton")}
+    texts = [e.text for e in elements if e.text]
+
+    if not hasattr(device, "screens"):
+        return PhoneState(source="device", switches=switches, texts=texts,
+                          settings=adb_settings(device))
+
+    return PhoneState(
+        source="sim",
+        screen=device.current,
+        settings={k: bool(v) for k, v in device.state.items()},
+        switches=switches,
+        texts=texts,
+        typed=list(device.typed),
+        fields=dict(getattr(device, "fields", {})),
+        records={k: [dict(r) for r in v]
+                 for k, v in getattr(device, "records", {}).items()},
+        toggled=list(getattr(device, "toggled", [])),
+    )
+
+
+ADB_SETTINGS = {
+    "wifi": ("global", "wifi_on"),
+    "airplane": ("global", "airplane_mode_on"),
+    "bluetooth": ("global", "bluetooth_on"),
+}
+"""Settings readable out of band, so scoring does not depend on which screen
+the agent left the phone on."""
+
+
+def adb_settings(device) -> dict[str, bool]:
+    """Read the authoritative values from a real phone. Missing keys are
+    omitted rather than defaulted -- an unreadable setting must not be scored
+    as "off"."""
+    out: dict[str, bool] = {}
+    for key, (namespace, name) in ADB_SETTINGS.items():
+        try:
+            raw = device._run("shell", "settings", "get", namespace, name)
+        except Exception:
+            continue
+        raw = raw.strip()
+        if raw in ("0", "1"):
+            out[key] = raw == "1"
+    return out

benchmarks/edge_harness/android_agent/run_agent.py

@@ -0,0 +1,98 @@
+"""Drive Spark-X2.5 through Android control steps against an OpenAI endpoint.
+
+Measures whether the model picks the right tool and the right element, and how
+long a decision takes -- the two things that decide whether a 1.7B model can
+actually run a phone agent locally.
+"""
+import argparse, json, time
+from urllib import request as urlrequest
+
+from action_space import SYSTEM_PROMPT, TOOLS
+from scenarios import SCENARIOS
+
+
+def chat(base_url, model, messages, tools, timeout=900, max_tokens=1024,
+         think=True):
+    payload = {
+        "model": model, "messages": messages, "tools": tools,
+        "tool_choice": "auto", "temperature": 0.0, "max_tokens": max_tokens,
+    }
+    if not think:
+        # Spark's template opens <think> by default; closing it up front makes
+        # the model answer directly, which is what a phone agent wants.
+        payload["chat_template_kwargs"] = {"enable_thinking": False}
+    body = json.dumps(payload).encode()
+    req = urlrequest.Request(f"{base_url}/chat/completions", data=body,
+                             headers={"Content-Type": "application/json"})
+    t0 = time.perf_counter()
+    with urlrequest.urlopen(req, timeout=timeout) as r:
+        out = json.loads(r.read())
+    return out, time.perf_counter() - t0
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--base-url", default="http://127.0.0.1:8077/v1")
+    ap.add_argument("--model", default="XHToken/Spark-X2.5-1.7B")
+    ap.add_argument("--no-think", action="store_true")
+    ap.add_argument("--max-tokens", type=int, default=1024)
+    ap.add_argument("--out", default="android_agent_results.json")
+    a = ap.parse_args()
+
+    rows, correct_tool, correct_args = [], 0, 0
+    for sc in SCENARIOS:
+        msgs = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content":
+                f"Task: {sc['task']}\n\nCurrent screen:\n{sc['screen']}\n\n"
+                "Choose the next action."},
+        ]
+        try:
+            out, dt = chat(a.base_url, a.model, msgs, TOOLS,
+                           max_tokens=a.max_tokens, think=not a.no_think)
+        except Exception as e:
+            rows.append({"id": sc["id"], "error": str(e)[:200]})
+            print(f"{sc['id']:<20} ERROR {str(e)[:90]}", flush=True)
+            continue
+        msg = out["choices"][0]["message"]
+        calls = msg.get("tool_calls") or []
+        got_name = calls[0]["function"]["name"] if calls else None
+        try:
+            got_args = json.loads(calls[0]["function"]["arguments"]) if calls else {}
+        except Exception:
+            got_args = {"_raw": calls[0]["function"]["arguments"]}
+
+        name_ok = got_name in sc["accept_names"]
+        args_ok = name_ok and all(
+            str(got_args.get(k)) == str(v) for k, v in sc["expect"]["args"].items()
+        ) if got_name == sc["expect"]["name"] else name_ok and not sc["expect"]["args"]
+        correct_tool += int(name_ok)
+        correct_args += int(bool(args_ok))
+        rows.append({
+            "id": sc["id"], "latency_s": round(dt, 2),
+            "expected": sc["expect"], "got_name": got_name, "got_args": got_args,
+            "tool_ok": name_ok, "args_ok": bool(args_ok),
+            "reasoning_chars": len(msg.get("reasoning_content") or ""),
+            "completion_tokens": out.get("usage", {}).get("completion_tokens"),
+        })
+        flag = "OK " if args_ok else ("~  " if name_ok else "X  ")
+        print(f"{flag}{sc['id']:<20} {got_name}({got_args}) {dt:.1f}s", flush=True)
+
+    n = len(SCENARIOS)
+    summary = {
+        "scenarios": n,
+        "thinking": not a.no_think,
+        "tool_accuracy": f"{correct_tool}/{n}",
+        "action_accuracy": f"{correct_args}/{n}",
+        "median_latency_s": round(
+            sorted(r["latency_s"] for r in rows if "latency_s" in r)[
+                max(0, len([r for r in rows if "latency_s" in r]) // 2)] , 2)
+        if any("latency_s" in r for r in rows) else None,
+        "rows": rows,
+    }
+    json.dump(summary, open(a.out, "w"), indent=2)
+    print(json.dumps({k: v for k, v in summary.items() if k != "rows"}, indent=2))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/android_agent/run_episode.py

@@ -0,0 +1,106 @@
+"""Drive Spark-X2.5 through a whole phone task, not one decision.
+
+    # against the in-process simulated phone (no device needed)
+    python run_episode.py --task "Turn on Wi-Fi." --device sim
+
+    # against a real phone
+    adb devices                      # confirm it is listed
+    python run_episode.py --task "Turn on Wi-Fi." --device adb
+
+The server side is vLLM's stock OpenAI endpoint; Spark emits GLM-4.5's tool
+format, so no custom parser is needed:
+
+    vllm serve <model> --port 8077 --enable-auto-tool-choice \\
+        --tool-call-parser glm45 --reasoning-parser glm45
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+
+from agent import AgentConfig, run_episode
+from device import AdbDevice, AdbError, demo_phone
+
+TASKS = {
+    "wifi": "Turn on Wi-Fi.",
+    "alarm": "Set an alarm for 7:30 AM.",
+}
+
+
+def build_device(kind: str, serial: str | None):
+    if kind == "sim":
+        return demo_phone(), "simulated phone (settings + clock)"
+    try:
+        device = AdbDevice(serial=serial)
+        w, h = device.size()
+        return device, f"adb device {serial or '(default)'} {w}x{h}"
+    except AdbError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        raise SystemExit(2)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--task", default=TASKS["wifi"],
+                    help=f"free text, or one of: {', '.join(TASKS)}")
+    ap.add_argument("--device", choices=("sim", "adb"), default="sim")
+    ap.add_argument("--serial", default=None)
+    ap.add_argument("--base-url", default="http://127.0.0.1:8077/v1")
+    ap.add_argument("--model", default="XHToken/Spark-X2.5-1.7B")
+    ap.add_argument("--max-steps", type=int, default=12)
+    ap.add_argument("--max-tokens", type=int, default=1024)
+    ap.add_argument("--no-think", action="store_true")
+    ap.add_argument("--no-think-fallback", action="store_true")
+    ap.add_argument("--out", default=None)
+    a = ap.parse_args()
+
+    task = TASKS.get(a.task, a.task)
+    device, described = build_device(a.device, a.serial)
+    cfg = AgentConfig(
+        base_url=a.base_url, model=a.model, max_steps=a.max_steps,
+        max_tokens=a.max_tokens, think=not a.no_think,
+        think_fallback=not a.no_think_fallback,
+    )
+
+    print(f"device: {described}")
+    print(f"task:   {task}")
+    t0 = time.perf_counter()
+    episode = run_episode(device, task, cfg)
+    wall = time.perf_counter() - t0
+
+    print()
+    print(episode.transcript())
+    latencies = [s.latency_s for s in episode.steps]
+    print(f"\n{episode.n_steps} steps in {wall:.1f} s"
+          f"  (median decision {sorted(latencies)[len(latencies) // 2]:.2f} s)"
+          if latencies else f"\n{episode.n_steps} steps in {wall:.1f} s")
+    if hasattr(device, "state"):
+        print(f"device state: {device.state}")
+
+    if a.out:
+        payload = {
+            "task": task, "device": described, "done": episode.done,
+            "summary": episode.summary, "stop_reason": episode.stop_reason,
+            "wall_s": round(wall, 2),
+            "think": cfg.think, "max_steps": cfg.max_steps,
+            "steps": [
+                {"n": s.n, "tool": s.tool, "args": s.args,
+                 "ok": s.result.ok if s.result else False,
+                 "message": s.result.message if s.result else s.note,
+                 "latency_s": round(s.latency_s, 3), "thinking": s.thinking}
+                for s in episode.steps
+            ],
+            "final_state": getattr(device, "state", None),
+        }
+        with open(a.out, "w") as f:
+            json.dump(payload, f, indent=2, ensure_ascii=False)
+        print(f"wrote {a.out}")
+
+    raise SystemExit(0 if episode.done else 1)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/android_agent/run_suite.py

@@ -0,0 +1,151 @@
+"""Run the task suite and score it, so an agent change can be compared.
+
+Success is decided by the phone's own settings and screen, not by the
+transcript: an agent that says it turned Wi-Fi on and left it off fails here,
+which is exactly the failure this model exhibits.
+
+Two things changed after the plan review:
+
+* ``--repeat N`` now walks the task's **variants** rather than re-running one
+  fixture. At temperature 0 against fixed fixtures, repeats measure
+  repeatability; they only become trials if the phone differs between them.
+  The summary reports a pass rate per task with the variants named, so a build
+  comparison rests on more than one cell.
+* ``--device adb`` scores a real phone. Each run resets the device to the
+  task's declared starting state first, and a task that cannot be scored from
+  a phone (because its predicate needs flip history) is skipped and reported
+  as skipped, rather than silently scored from final state.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import time
+from collections import defaultdict
+
+from agent import AgentConfig, run_episode, wait_for_server
+from phone_state import read_state
+from tasks import SUITE, SUITE_BY_ID
+
+
+def _reset_real_device(device, task) -> None:
+    if not task.reset_adb:
+        raise SystemExit(
+            f"task {task.task_id!r} declares no reset_adb, so a phone run would "
+            f"be scored against an unknown starting state. Add one or exclude "
+            f"the task with --tasks."
+        )
+    for cmd in task.reset_adb:
+        device._run(*cmd)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--base-url", default="http://127.0.0.1:8077/v1")
+    ap.add_argument("--model", default="spark")
+    ap.add_argument("--tasks", default="all",
+                    help="comma-separated task ids, or 'all'")
+    ap.add_argument("--repeat", type=int, default=1,
+                    help="runs per task; each walks the next variant")
+    ap.add_argument("--device", choices=("sim", "adb"), default="sim")
+    ap.add_argument("--serial", default=None, help="adb device serial")
+    ap.add_argument("--no-think", action="store_true")
+    ap.add_argument("--max-tokens", type=int, default=1024)
+    ap.add_argument("--label", default="")
+    ap.add_argument("--out", default=None)
+    a = ap.parse_args()
+
+    tasks = SUITE if a.tasks == "all" else [SUITE_BY_ID[t] for t in a.tasks.split(",")]
+    cfg = AgentConfig(base_url=a.base_url, model=a.model,
+                      think=not a.no_think, max_tokens=a.max_tokens)
+
+    phone = None
+    if a.device == "adb":
+        from device import AdbDevice
+        phone = AdbDevice(serial=a.serial)
+        skipped = [t.task_id for t in tasks if t.needs]
+        if skipped:
+            print(f"skipping on a real phone (needs flip history): "
+                  f"{', '.join(skipped)}")
+        tasks = [t for t in tasks if not t.needs]
+
+    if not wait_for_server(cfg, timeout_s=600):
+        raise SystemExit(f"no model server at {a.base_url}")
+
+    rows, t0 = [], time.perf_counter()
+    for task in tasks:
+        for run in range(a.repeat):
+            if phone is not None:
+                _reset_real_device(phone, task)
+                device = phone
+            else:
+                device = task.build(run)
+            cfg.max_steps = task.max_steps
+            episode = run_episode(device, task.prompt, cfg)
+            score = task.score(read_state(device), episode.done, run)
+            score.update(
+                run=run + 1, steps=episode.n_steps,
+                stop_reason=episode.stop_reason,
+                recovered_steps=sum(1 for s in episode.steps
+                                    if s.parsed_via not in ("tool_calls", "none")),
+                median_latency_s=round(
+                    sorted(s.latency_s for s in episode.steps)[episode.n_steps // 2], 2)
+                if episode.n_steps else None,
+            )
+            rows.append(score)
+            mark = "PASS" if score["success"] else "fail"
+            detail = []
+            if not score.get("scorable", True):
+                mark, detail = "skip", [score.get("reason", "")]
+            else:
+                if score["reached_goal"] and not score["declared_done"]:
+                    detail.append("goal reached but never stopped")
+                if score["declared_done"] and not score["reached_goal"]:
+                    detail.append("claimed done, goal NOT reached")
+                if score["collateral_damage"]:
+                    detail.append("collateral damage")
+            print(f"  {mark}  {task.task_id:18s} {score['variant']:10s} "
+                  f"steps={score['steps']:2d} "
+                  f"{score['stop_reason'][:34]:36s} {'; '.join(detail)}", flush=True)
+
+    wall = time.perf_counter() - t0
+    scored = [r for r in rows if r.get("scorable", True)]
+    n_pass = sum(1 for r in scored if r["success"])
+    reached = sum(1 for r in scored if r["reached_goal"])
+    claimed = sum(1 for r in scored if r["declared_done"] and not r["reached_goal"])
+
+    by_task: dict[str, list[dict]] = defaultdict(list)
+    for r in scored:
+        by_task[r["task_id"]].append(r)
+    print("\npass rate per task")
+    for task_id, rs in by_task.items():
+        passes = sum(1 for r in rs if r["success"])
+        failed = [r["variant"] for r in rs if not r["success"]]
+        print(f"  {task_id:18s} {passes}/{len(rs)}"
+              + (f"   failed on: {', '.join(failed)}" if failed else ""))
+
+    rates = [sum(1 for r in rs if r["success"]) / len(rs) for rs in by_task.values()]
+    spread = (f"{min(rates):.0%}-{max(rates):.0%}" if rates else "-")
+    print(f"\n{n_pass}/{len(scored)} passed in {wall:.0f}s"
+          f"   (goal reached {reached}/{len(scored)}; "
+          f"falsely claimed done {claimed}; per-task rate {spread}"
+          + (f"; median {statistics.median(rates):.0%}" if rates else "") + ")")
+    if len(scored) and len(by_task) and max(len(rs) for rs in by_task.values()) < 3:
+        print("  NOTE: fewer than 3 runs per task -- enough to show a failure "
+              "reproduces, not to rank builds.")
+
+    if a.out:
+        with open(a.out, "w") as f:
+            json.dump({"label": a.label, "base_url": a.base_url,
+                       "model": a.model, "think": cfg.think,
+                       "device": a.device, "repeat": a.repeat,
+                       "passed": n_pass, "total": len(scored),
+                       "skipped": len(rows) - len(scored),
+                       "wall_s": round(wall, 1), "rows": rows}, f, indent=2)
+        print(f"wrote {a.out}")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/android_agent/scenarios.py

@@ -0,0 +1,113 @@
+"""Android screens as a text-only agent sees them, with the correct next action.
+
+Each screen is a uiautomator-style accessibility dump flattened to one line per
+element -- the same shape droidrun's Portal and AndroidWorld's a11y observation
+hand to a text model. `expect` is the action a competent agent must take next;
+`accept_names` allows a defensible alternative where one genuinely exists.
+"""
+
+SCENARIOS = [
+    {
+        "id": "launch_clock",
+        "task": "Set an alarm for 7:30 AM.",
+        "screen": """[0] TextView "Phone" clickable=true
+[1] TextView "Messages" clickable=true
+[2] TextView "Chrome" clickable=true
+[3] TextView "Clock" clickable=true
+[4] TextView "Settings" clickable=true
+[5] TextView "Camera" clickable=true""",
+        "expect": {"name": "tap", "args": {"index": 3}},
+        "accept_names": ["tap", "open_app"],
+    },
+    {
+        "id": "clock_alarm_tab",
+        "task": "Set an alarm for 7:30 AM.",
+        "screen": """[0] Button "Alarm" clickable=true selected=false
+[1] Button "Clock" clickable=true selected=true
+[2] Button "Timer" clickable=true selected=false
+[3] Button "Stopwatch" clickable=true selected=false
+[4] TextView "14:32" clickable=false""",
+        "expect": {"name": "tap", "args": {"index": 0}},
+        "accept_names": ["tap"],
+    },
+    {
+        "id": "add_alarm",
+        "task": "Set an alarm for 7:30 AM.",
+        "screen": """[0] Button "Alarm" clickable=true selected=true
+[1] TextView "No alarms" clickable=false
+[2] ImageButton "Add alarm" clickable=true
+[3] Button "Timer" clickable=true""",
+        "expect": {"name": "tap", "args": {"index": 2}},
+        "accept_names": ["tap"],
+    },
+    {
+        "id": "wifi_settings",
+        "task": "Turn on Wi-Fi.",
+        "screen": """[0] TextView "Settings" clickable=false
+[1] TextView "Network & internet" clickable=true
+[2] TextView "Connected devices" clickable=true
+[3] TextView "Apps" clickable=true
+[4] TextView "Battery" clickable=true
+[5] TextView "Display" clickable=true""",
+        "expect": {"name": "tap", "args": {"index": 1}},
+        "accept_names": ["tap"],
+    },
+    {
+        "id": "wifi_toggle",
+        "task": "Turn on Wi-Fi.",
+        "screen": """[0] TextView "Network & internet" clickable=false
+[1] Switch "Wi-Fi" checked=false clickable=true
+[2] TextView "Mobile network" clickable=true
+[3] Switch "Airplane mode" checked=false clickable=true""",
+        "expect": {"name": "tap", "args": {"index": 1}},
+        "accept_names": ["tap"],
+    },
+    {
+        "id": "search_contact",
+        "task": "Find the contact named Wei Zhang.",
+        "screen": """[0] EditText "Search contacts" clickable=true focused=true
+[1] TextView "Alice Chen" clickable=true
+[2] TextView "Bob Lee" clickable=true
+[3] TextView "Carol Wu" clickable=true""",
+        "expect": {"name": "type_text", "args": {"index": 0, "text": "Wei Zhang"}},
+        "accept_names": ["type_text"],
+    },
+    {
+        "id": "scroll_for_item",
+        "task": "Open the Developer options screen in Settings.",
+        "screen": """[0] TextView "Settings" clickable=false
+[1] TextView "Display" clickable=true
+[2] TextView "Sound" clickable=true
+[3] TextView "Storage" clickable=true
+[4] TextView "Privacy" clickable=true
+[5] TextView "Location" clickable=true""",
+        "expect": {"name": "swipe", "args": {"direction": "up"}},
+        "accept_names": ["swipe"],
+    },
+    {
+        "id": "go_back",
+        "task": "Return to the previous screen; you opened the wrong menu.",
+        "screen": """[0] TextView "About phone" clickable=false
+[1] TextView "Model: Pixel 8" clickable=false
+[2] TextView "Android version" clickable=true""",
+        "expect": {"name": "press_key", "args": {"key": "back"}},
+        "accept_names": ["press_key"],
+    },
+    {
+        "id": "open_app_by_name",
+        "task": "Open the Camera app.",
+        "screen": """[0] TextView "Swipe up to unlock" clickable=false
+[1] TextView "10:04" clickable=false""",
+        "expect": {"name": "open_app", "args": {"name": "Camera"}},
+        "accept_names": ["open_app", "swipe"],
+    },
+    {
+        "id": "task_done",
+        "task": "Turn on Wi-Fi.",
+        "screen": """[0] TextView "Network & internet" clickable=false
+[1] Switch "Wi-Fi" checked=true clickable=true
+[2] TextView "Connected to HOME-5G" clickable=false""",
+        "expect": {"name": "task_complete", "args": {}},
+        "accept_names": ["task_complete"],
+    },
+]

benchmarks/edge_harness/android_agent/tasks.py

@@ -0,0 +1,279 @@
+"""A scored task suite, judged by what the phone shows rather than by what the
+agent said -- or by what the simulator happens to remember.
+
+Three things were wrong with the first version of this file, and all three
+made it score more generously than it claimed to:
+
+* **``add_alarm`` passed on any typed string containing "7".** Typing ``7``
+  and stopping scored a success on a task whose prompt is "Set an alarm for
+  7:30 AM": no minute, no meridiem, and no requirement that the alarm was ever
+  saved. It now requires a *committed* alarm reading 7:30 AM, which the clock
+  screen lists, so the check is one a real phone can also answer.
+* **``wifi_on_already`` scored final state only.** Its note said "any tap on
+  the switch fails the task"; the code could not see a tap, so turning Wi-Fi
+  off and back on passed. Switch flips are now recorded and the task fails on
+  any of them -- which is the exact failure W4A8 exhibited (it turned Wi-Fi on
+  and then tapped the switch off again).
+* **Predicates read ``SimulatedDevice`` attributes**, so every scored run was a
+  simulator run by construction. They read :class:`PhoneState` now, which comes
+  from the UI dump for both backends.
+
+Repeats are only trials if something differs between them, so each task
+carries **variants**: the same goal against a renamed control, a reordered
+screen, or a different starting state. Five runs of one fixture at temperature
+0 measure repeatability; five variants measure whether the agent can do the
+task.
+"""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass, field
+from typing import Callable
+
+from device import SimElement, SimulatedDevice, demo_phone
+from phone_state import PhoneState, read_state
+
+Predicate = Callable[[PhoneState], bool]
+
+CLOCK_PACKAGE = "com.google.android.deskclock"
+"""Clock app to reset for the alarm tasks. AOSP builds and several OEMs ship a
+different package (``com.android.deskclock``, ``com.sec.android.app.clockpackage``
+on Samsung); override before a real-device run, because clearing the wrong
+package leaves a stale alarm and the next run scores against it."""
+
+
+# --------------------------------------------------------------- variations
+
+def rename(screen: str, old: str, new: str) -> Callable[[SimulatedDevice], None]:
+    """Same control, different word. Android OEMs really do ship "WLAN"."""
+    def apply(d: SimulatedDevice) -> None:
+        for el in d.screens[screen].elements:
+            if el.label == old:
+                el.label = new
+    return apply
+
+
+def reorder(screen: str, seed: int) -> Callable[[SimulatedDevice], None]:
+    """Shuffle the tappable rows. Index-based tapping must not be luck."""
+    def apply(d: SimulatedDevice) -> None:
+        els = d.screens[screen].elements
+        head = [e for e in els if not e.clickable]
+        body = [e for e in els if e.clickable]
+        random.Random(seed).shuffle(body)
+        d.screens[screen].elements = head + body
+    return apply
+
+
+def decoy(screen: str, label: str) -> Callable[[SimulatedDevice], None]:
+    """A plausible wrong row, e.g. "Wi-Fi calling" next to "Wi-Fi"."""
+    def apply(d: SimulatedDevice) -> None:
+        d.screens[screen].elements.append(SimElement(label))
+    return apply
+
+
+def chain(*fns: Callable[[SimulatedDevice], None]) -> Callable[[SimulatedDevice], None]:
+    def apply(d: SimulatedDevice) -> None:
+        for fn in fns:
+            fn(d)
+    return apply
+
+
+@dataclass
+class Variant:
+    variant_id: str
+    mutate: Callable[[SimulatedDevice], None] = lambda d: None
+
+
+# --------------------------------------------------------------------- task
+
+@dataclass
+class Task:
+    task_id: str
+    prompt: str
+    reaches: Predicate
+    must_not: Predicate = lambda s: False
+    setup: Callable[[SimulatedDevice], None] = lambda d: None
+    max_steps: int = 10
+    note: str = ""
+    variants: list[Variant] = field(default_factory=lambda: [Variant("base")])
+    needs: tuple[str, ...] = ()
+    """PhoneState facts this task's predicates require that a screen cannot
+    show (``toggled``, ``fields``, ``records``). A task listing any of these
+    cannot be scored on a real phone, and says so instead of scoring wrong."""
+    reset_adb: tuple[tuple[str, ...], ...] = ()
+    """Commands that put a real phone into this task's starting state. Without
+    them a real-device run is scored against an unknown start, so an empty
+    tuple means "refuse to run me on a phone"."""
+
+    def variant(self, n: int) -> Variant:
+        return self.variants[n % len(self.variants)]
+
+    def build(self, run: int = 0) -> SimulatedDevice:
+        device = demo_phone()
+        self.variant(run).mutate(device)
+        self.setup(device)
+        return device
+
+    def scorable_on(self, state: PhoneState) -> str | None:
+        """Why this task cannot be scored from ``state``, or None if it can."""
+        if state.source == "device" and self.needs:
+            return (f"needs {', '.join(self.needs)}, which a screen cannot "
+                    f"show; simulator only")
+        return None
+
+    def score(self, device_or_state, declared_done: bool, run: int = 0) -> dict:
+        state = (device_or_state if isinstance(device_or_state, PhoneState)
+                 else read_state(device_or_state))
+        blocked = self.scorable_on(state)
+        if blocked:
+            return {"task_id": self.task_id, "variant": self.variant(run).variant_id,
+                    "scorable": False, "reason": blocked, "success": False,
+                    "reached_goal": None, "collateral_damage": None,
+                    "declared_done": declared_done}
+        reached = bool(self.reaches(state))
+        damaged = bool(self.must_not(state))
+        return {
+            "task_id": self.task_id,
+            "variant": self.variant(run).variant_id,
+            "scorable": True,
+            "reached_goal": reached,
+            "collateral_damage": damaged,
+            "declared_done": declared_done,
+            # The only row that counts: the phone is in the requested state,
+            # nothing else was broken, and the agent knew to stop.
+            "success": reached and not damaged and declared_done,
+        }
+
+
+# ----------------------------------------------------------------- oracles
+
+# Both the on-screen labels and the setting key, because `switch()` matches
+# either source: "Wi-Fi" normalises onto the `wifi` setting by luck, but
+# "Airplane mode" does not normalise onto `airplane`, and a name the setting
+# lookup misses silently falls through to "control not on screen".
+WIFI = ("Wi-Fi", "WiFi", "WLAN", "wifi")
+AIRPLANE = ("Airplane mode", "Flight mode", "airplane")
+
+
+def _wifi_is(want: bool) -> Predicate:
+    # `switch` returns None when the control is not on screen, which is not
+    # the same as off: an agent that never navigated there has not turned
+    # anything on.
+    return lambda s: s.switch(*WIFI) is want
+
+
+def _airplane_on(s: PhoneState) -> bool:
+    return s.switch(*AIRPLANE) is True
+
+
+def _alarm_set(hour: int, minute: int, pm: bool) -> Predicate:
+    """A *saved* alarm reading the requested time.
+
+    The clock screen lists committed alarms, so this reads the screen and works
+    on a real phone. Typing the digits without saving leaves the list empty.
+    """
+    wanted = f"{hour}:{minute:02d} {'PM' if pm else 'AM'}"
+    return lambda s: s.has_text(wanted)
+
+
+SUITE: list[Task] = [
+    Task(
+        task_id="wifi_on",
+        prompt="Turn on Wi-Fi.",
+        reaches=_wifi_is(True),
+        must_not=_airplane_on,
+        note="Two screens deep, then a switch. The switch's state is visible, "
+             "so stopping is decidable from the screen.",
+        variants=[
+            Variant("base"),
+            Variant("wlan", rename("network", "Wi-Fi", "WLAN")),
+            Variant("reordered", reorder("network", seed=7)),
+            Variant("decoy", decoy("network", "Wi-Fi calling")),
+            Variant("deep", chain(reorder("settings", seed=3),
+                                  decoy("network", "Wi-Fi preferences"))),
+        ],
+        reset_adb=(("shell", "svc", "wifi", "disable"),),
+    ),
+    Task(
+        task_id="wifi_on_already",
+        prompt="Turn on Wi-Fi.",
+        setup=lambda d: d.state.__setitem__("wifi", True),
+        reaches=lambda s: s.switch(*WIFI) is True and s.times_toggled("wifi") == 0,
+        must_not=_airplane_on,
+        note="Already satisfied. The only correct action is task_complete. "
+             "Any flip of the switch fails, including off-and-on again, which "
+             "final state cannot see and which W4A8 actually did.",
+        needs=("toggled",),
+        variants=[
+            Variant("base"),
+            Variant("wlan", rename("network", "Wi-Fi", "WLAN")),
+            Variant("reordered", reorder("network", seed=11)),
+        ],
+    ),
+    Task(
+        task_id="wifi_off",
+        prompt="Turn off Wi-Fi.",
+        setup=lambda d: d.state.__setitem__("wifi", True),
+        reaches=_wifi_is(False),
+        must_not=_airplane_on,
+        note="The mirror of wifi_on: guards against a model that has simply "
+             "learned to tap switches.",
+        variants=[
+            Variant("base"),
+            Variant("wlan", rename("network", "Wi-Fi", "WLAN")),
+            Variant("reordered", reorder("network", seed=5)),
+        ],
+        reset_adb=(("shell", "svc", "wifi", "enable"),),
+    ),
+    Task(
+        task_id="airplane_on",
+        prompt="Turn on airplane mode.",
+        reaches=_airplane_on,
+        note="Same screen as Wi-Fi, different row -- catches index confusion, "
+             "which is why one variant reorders that screen.",
+        variants=[
+            Variant("base"),
+            Variant("reordered", reorder("network", seed=2)),
+            Variant("decoy", decoy("network", "Airplane mode settings")),
+        ],
+        # `settings put global` needs WRITE_SECURE_SETTINGS, which adb has on a
+        # debuggable build but not on every retail phone; the run fails loudly
+        # if it is refused rather than scoring against an unknown start.
+        reset_adb=(("shell", "settings", "put", "global", "airplane_mode_on", "0"),),
+    ),
+    Task(
+        task_id="open_clock",
+        prompt="Open the Clock app and go to the Alarm tab.",
+        reaches=lambda s: s.has_text("Add alarm") or s.has_text("Set time"),
+        note="Navigation only, no state change. Isolates 'can it find an app' "
+             "from 'can it finish'.",
+        variants=[
+            Variant("base"),
+            Variant("reordered", reorder("home", seed=4)),
+            Variant("decoy", decoy("home", "Clock Widget")),
+        ],
+        reset_adb=(("shell", "am", "force-stop", CLOCK_PACKAGE),
+                   ("shell", "input", "keyevent", "KEYCODE_HOME")),
+    ),
+    Task(
+        task_id="add_alarm",
+        prompt="Set an alarm for 7:30 AM.",
+        reaches=_alarm_set(7, 30, pm=False),
+        must_not=lambda s: s.has_text("7:30 PM"),
+        max_steps=12,
+        note="The full flow: right app, right tab, add, type both fields, "
+             "leave the meridiem alone, save. Previously this passed on any "
+             "typed string containing '7'.",
+        variants=[
+            Variant("base"),
+            Variant("reordered", reorder("clock", seed=9)),
+            Variant("decoy", decoy("alarms", "Alarm settings")),
+        ],
+        # `pm clear` removes saved alarms, so run N does not see run N-1's.
+        reset_adb=(("shell", "pm", "clear", CLOCK_PACKAGE),
+                   ("shell", "input", "keyevent", "KEYCODE_HOME")),
+    ),
+]
+
+SUITE_BY_ID = {t.task_id: t for t in SUITE}

benchmarks/edge_harness/android_agent/test_android_agent.py

@@ -0,0 +1,525 @@
+"""Tests for the Android agent toolkit.
+
+No phone, emulator or model server is needed: the device is the in-process
+simulator and the model is a scripted stand-in. That is deliberate -- this
+host has no `adb`, no emulator and no `/dev/kvm`, so anything that could only
+be checked against real hardware would otherwise ship unverified.
+"""
+
+import json
+
+import pytest
+
+import agent as agent_mod
+from agent import AgentConfig, first_tool_call, run_episode
+from device import demo_phone
+from executor import execute
+from ui_tree import parse_ui_dump, render_tree, screen_signature
+
+# --------------------------------------------------------------------- ui_tree
+
+REAL_DUMP = """<?xml version="1.0" encoding="UTF-8"?>
+<hierarchy rotation="0">
+ <node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.android.settings" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1080,2400]">
+  <node index="1" text="Network &amp; internet" resource-id="android:id/title" class="android.widget.TextView" package="com.android.settings" content-desc="" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,300][1080,460]" />
+  <node index="2" text="" resource-id="" class="android.widget.LinearLayout" package="com.android.settings" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,460][1080,620]" />
+  <node index="3" text="" resource-id="com.android.settings:id/switch_widget" class="android.widget.Switch" package="com.android.settings" content-desc="Wi-Fi" checkable="true" checked="true" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[800,460][1000,620]" />
+ </node>
+</hierarchy>"""
+
+
+def test_parses_a_real_dump_and_keeps_only_what_is_actionable():
+    els = parse_ui_dump(REAL_DUMP)
+    labels = [e.text for e in els]
+    assert "Network & internet" in labels          # entity decoded
+    assert "Wi-Fi" in labels                        # fell back to content-desc
+    assert "" not in labels                         # bare layout node dropped
+    assert [e.index for e in els] == list(range(len(els)))
+
+
+def test_switch_state_is_visible_to_the_model():
+    """The one clearly wrong action measured earlier was tapping a switch that
+    was already on; that is only avoidable if the state is on the screen."""
+    rendered = render_tree(parse_ui_dump(REAL_DUMP))
+    assert "checked=true" in rendered
+    assert 'Switch "Wi-Fi"' in rendered
+
+
+def test_index_maps_to_the_centre_of_the_element():
+    els = parse_ui_dump(REAL_DUMP)
+    wifi = next(e for e in els if e.text == "Wi-Fi")
+    assert wifi.center == (900, 540)
+
+
+def test_signature_tracks_state_not_pixels():
+    a = parse_ui_dump(REAL_DUMP)
+    shifted = REAL_DUMP.replace("[0,300][1080,460]", "[0,310][1080,470]")
+    assert screen_signature(parse_ui_dump(shifted)) == screen_signature(a)
+    toggled = REAL_DUMP.replace('checkable="true" checked="true"',
+                                'checkable="true" checked="false"')
+    assert screen_signature(parse_ui_dump(toggled)) != screen_signature(a)
+
+
+def test_element_cap_bounds_the_context():
+    big = '<?xml version="1.0"?><hierarchy>' + "".join(
+        f'<node text="row{i}" class="android.widget.TextView" clickable="true" '
+        f'bounds="[0,{i},100,{i + 10}]" />' for i in range(200)
+    ) + "</hierarchy>"
+    assert len(parse_ui_dump(big, max_elements=25)) == 25
+
+
+# -------------------------------------------------------------------- executor
+
+def test_tap_lands_on_the_right_row_and_reports_prior_state():
+    device = demo_phone()
+    device.current = "network"
+    els = parse_ui_dump(device.dump())
+    wifi = next(e for e in els if e.text == "Wi-Fi")
+    result = execute("tap", {"index": wifi.index}, els, device)
+    assert result.ok and device.state["wifi"] is True
+    assert "was off" in result.message
+
+
+def test_out_of_range_index_is_a_message_not_an_exception():
+    device = demo_phone()
+    els = parse_ui_dump(device.dump())
+    result = execute("tap", {"index": 99}, els, device)
+    assert not result.ok
+    assert "out of range" in result.message and f"0-{len(els) - 1}" in result.message
+
+
+def test_bad_arguments_explain_themselves():
+    device = demo_phone()
+    els = parse_ui_dump(device.dump())
+    assert "missing required argument" in execute("tap", {}, els, device).message
+    assert "whole number" in execute("tap", {"index": "banana"}, els, device).message
+    assert "unknown tool" in execute("frobnicate", {}, els, device).message
+    assert "direction must be" in execute("swipe", {"direction": "sideways"}, els, device).message
+
+
+def test_type_text_focuses_the_field_first():
+    device = demo_phone()
+    device.current = "new_alarm"
+    els = parse_ui_dump(device.dump())
+    field = next(e for e in els if e.text == "Hour")
+    execute("type_text", {"index": field.index, "text": "07"}, els, device)
+    kinds = [a[0] for a in device.actions]
+    assert kinds == ["tap", "text"], kinds
+    assert device.typed == ["07"]
+
+
+def test_swipe_direction_is_content_movement():
+    device = demo_phone()
+    els = parse_ui_dump(device.dump())
+    execute("swipe", {"direction": "down"}, els, device)
+    _, x1, y1, x2, y2 = device.actions[-1]
+    assert int(y1) > int(y2), "scrolling down must move the finger up"
+
+
+def test_task_complete_carries_its_summary():
+    device = demo_phone()
+    result = execute("task_complete", {"summary": "Wi-Fi is on"}, [], device)
+    assert result.done and result.summary == "Wi-Fi is on"
+
+
+# ----------------------------------------------------------------------- agent
+
+def _scripted(calls):
+    """A stand-in model that plays a fixed list of tool calls."""
+    seq = list(calls)
+
+    def fake_chat(cfg, messages, *, think):
+        call = seq.pop(0) if seq else None
+        if call is None:
+            return {"choices": [{"message": {"content": "", "tool_calls": []}}]}, 0.01
+        name, args = call
+        return {"choices": [{"message": {"tool_calls": [
+            {"function": {"name": name, "arguments": json.dumps(args)}}
+        ]}}]}, 0.01
+
+    return fake_chat
+
+
+def test_first_tool_call_handles_both_argument_encodings():
+    as_str = {"choices": [{"message": {"tool_calls": [
+        {"function": {"name": "tap", "arguments": '{"index": 2}'}}]}}]}
+    as_dict = {"choices": [{"message": {"tool_calls": [
+        {"function": {"name": "tap", "arguments": {"index": 2}}}]}}]}
+    assert first_tool_call(as_str) == ("tap", {"index": 2})
+    assert first_tool_call(as_dict) == ("tap", {"index": 2})
+    assert first_tool_call({"choices": [{"message": {"content": "hi"}}]}) == (None, {})
+    # Malformed JSON must not kill the episode.
+    broken = {"choices": [{"message": {"tool_calls": [
+        {"function": {"name": "tap", "arguments": "{not json"}}]}}]}
+    assert first_tool_call(broken) == ("tap", {})
+
+
+def test_completes_a_multi_screen_task(monkeypatch):
+    device = demo_phone()
+    monkeypatch.setattr(agent_mod, "chat", _scripted([
+        ("tap", {"index": 3}),                       # home -> Settings
+        ("tap", {"index": 1}),                       # -> Network & internet
+        ("tap", {"index": 1}),                       # Wi-Fi switch
+        ("task_complete", {"summary": "Wi-Fi enabled"}),
+    ]))
+    ep = run_episode(device, "Turn on Wi-Fi.", AgentConfig(max_steps=8))
+    assert ep.done and device.state["wifi"] is True
+    assert ep.summary == "Wi-Fi enabled"
+    assert ep.stop_reason == "task_complete"
+    assert ep.n_steps == 4
+
+
+def test_says_so_when_an_action_changes_nothing(monkeypatch):
+    device = demo_phone()
+    monkeypatch.setattr(agent_mod, "chat", _scripted([
+        ("tap", {"index": 0}),  # "Phone" on home: goes nowhere
+        ("task_complete", {"summary": "done"}),
+    ]))
+    ep = run_episode(device, "Do something.", AgentConfig(max_steps=4))
+    assert "did not change" in ep.steps[0].result.message
+
+
+def test_stops_when_it_repeats_a_no_op(monkeypatch):
+    device = demo_phone()
+    monkeypatch.setattr(agent_mod, "chat", _scripted([("tap", {"index": 0})] * 6))
+    ep = run_episode(device, "Do something.", AgentConfig(max_steps=6))
+    assert not ep.done
+    assert "stuck" in ep.stop_reason
+    assert ep.n_steps < 6, "should give up before the budget"
+
+
+def test_empty_reply_retries_without_thinking(monkeypatch):
+    """The measured failure: reasoning never closes `<think>`, so the parser
+    returns nothing. A non-thinking retry is worth ~0.6 s."""
+    seen = []
+
+    def fake_chat(cfg, messages, *, think):
+        seen.append(think)
+        if think:
+            return {"choices": [{"message": {"content": "", "tool_calls": []}}]}, 8.9
+        return {"choices": [{"message": {"tool_calls": [
+            {"function": {"name": "task_complete",
+                          "arguments": '{"summary": "fallback"}'}}]}}]}, 0.61
+
+    monkeypatch.setattr(agent_mod, "chat", fake_chat)
+    ep = run_episode(device := demo_phone(), "Task.", AgentConfig(max_steps=3))
+    assert seen == [True, False], seen
+    assert ep.done and ep.summary == "fallback"
+    assert ep.steps[0].thinking is False
+    assert device is not None
+
+
+def test_gives_up_rather_than_looping_when_no_tool_call_ever_comes(monkeypatch):
+    monkeypatch.setattr(agent_mod, "chat", _scripted([]))
+    ep = run_episode(demo_phone(), "Task.", AgentConfig(max_steps=5))
+    assert not ep.done
+    assert "no tool call" in ep.stop_reason
+    assert ep.n_steps == 1
+
+
+def test_step_budget_is_enforced(monkeypatch):
+    device = demo_phone()
+    # Alternate two real navigations so nothing is a repeat or a no-op.
+    monkeypatch.setattr(agent_mod, "chat", _scripted(
+        [("tap", {"index": 3}), ("press_key", {"key": "back"})] * 5))
+    ep = run_episode(device, "Wander.", AgentConfig(max_steps=4))
+    assert not ep.done
+    assert ep.n_steps == 4
+    assert "budget exhausted" in ep.stop_reason
+
+
+def test_transcript_is_readable(monkeypatch):
+    monkeypatch.setattr(agent_mod, "chat", _scripted([
+        ("tap", {"index": 3}), ("task_complete", {"summary": "ok"})]))
+    ep = run_episode(demo_phone(), "Open settings.", AgentConfig(max_steps=4))
+    text = ep.transcript()
+    assert "task: Open settings." in text
+    assert "tap(" in text and "task_complete" in text
+    assert "stopped: task_complete" in text
+
+
+def test_detects_undoing_its_own_action(monkeypatch):
+    """The measured failure: tapping a switch that is already on, turning it
+    back off, forever. Each tap really does change the screen, so the
+    no-change check cannot see it."""
+    device = demo_phone()
+    device.current = "network"
+    monkeypatch.setattr(agent_mod, "chat", _scripted([("tap", {"index": 1})] * 6))
+    ep = run_episode(device, "Turn on Wi-Fi.", AgentConfig(max_steps=6))
+    assert not ep.done
+    assert "undoing its own action" in ep.stop_reason
+    assert any("undid your previous action" in s.result.message
+               for s in ep.steps if s.result)
+
+
+def test_normal_navigation_is_not_mistaken_for_oscillation(monkeypatch):
+    """Opening a submenu and pressing back revisits a screen; that is how
+    phones work and must not trip the guard."""
+    device = demo_phone()
+    monkeypatch.setattr(agent_mod, "chat", _scripted([
+        ("tap", {"index": 3}),          # home -> settings
+        ("press_key", {"key": "back"}),  # -> home
+        ("tap", {"index": 2}),          # home -> clock
+        ("press_key", {"key": "back"}),  # -> home
+        ("task_complete", {"summary": "looked around"}),
+    ]))
+    ep = run_episode(device, "Look around.", AgentConfig(max_steps=8))
+    assert ep.done, ep.stop_reason
+    assert ep.n_steps == 5
+
+
+# ------------------------------------------------------------------ task suite
+
+def test_suite_scores_on_device_state_not_on_what_the_agent_said():
+    """The failure this model exhibits is claiming completion it did not
+    achieve, so the score must not read the transcript."""
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on"]
+    device = task.build()
+    assert task.score(device, declared_done=True)["success"] is False
+    device.state["wifi"] = True
+    assert task.score(device, declared_done=True)["success"] is True
+    # Reaching the goal without stopping is not success either.
+    assert task.score(device, declared_done=False)["success"] is False
+
+
+def test_collateral_damage_fails_the_task():
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on"]
+    device = task.build()
+    device.state.update(wifi=True, airplane=True)
+    score = task.score(device, declared_done=True)
+    assert score["reached_goal"] and score["collateral_damage"]
+    assert score["success"] is False
+
+
+def test_already_satisfied_task_starts_satisfied():
+    """Its only correct action is task_complete; tapping the switch fails."""
+    from phone_state import read_state
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on_already"]
+    device = task.build()
+    assert task.reaches(read_state(device))
+    assert task.score(device, declared_done=True)["success"] is True
+
+
+def test_already_satisfied_task_fails_on_a_switch_round_trip():
+    """W4A8 turned Wi-Fi on and then tapped the switch off again. Final state
+    cannot see that, so this task scored it a pass until the flips were
+    recorded."""
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on_already"]
+    device = task.build()
+    device.launch("Settings")
+    device.tap(540, 360)          # Network & internet
+    device.tap(540, 360)          # Wi-Fi off
+    device.tap(540, 360)          # ... and on again
+    assert device.state["wifi"] is True          # final state looks correct
+    assert task.score(device, declared_done=True)["success"] is False
+
+
+def test_goal_is_read_from_the_setting_not_from_the_final_screen():
+    """An agent that turns Wi-Fi on and then goes Home has still done it."""
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on"]
+    device = task.build()
+    device.launch("Settings")
+    device.tap(540, 360)
+    device.tap(540, 360)
+    device.keyevent("home")
+    assert task.score(device, declared_done=True)["success"] is True
+
+
+def test_never_finding_the_screen_is_not_scored_as_off():
+    """`switch()` returns None when neither the setting nor the screen knows
+    the control; a predicate must not read that as 'off' and pass a task."""
+    from phone_state import PhoneState
+    from tasks import SUITE_BY_ID
+
+    empty = PhoneState(source="device")
+    assert empty.switch("Wi-Fi") is None
+    assert SUITE_BY_ID["wifi_off"].score(empty, declared_done=True)["success"] is False
+
+
+def test_suite_includes_a_mirror_task_for_switch_tapping():
+    """Guards against a model that has simply learned 'tap the switch'."""
+    from tasks import SUITE_BY_ID
+
+    from phone_state import read_state
+
+    off = SUITE_BY_ID["wifi_off"]
+    device = off.build()
+    assert device.state["wifi"] is True
+    assert not off.reaches(read_state(device))
+    device.state["wifi"] = False
+    assert off.reaches(read_state(device))
+
+
+def test_chat_retries_transient_failures(monkeypatch):
+    """A phone task is a dozen round trips; one dropped connection should cost
+    a retry, not the episode."""
+    import agent as mod
+    from urllib import error as urlerror
+
+    calls = {"n": 0}
+
+    def flaky(req, timeout=None):
+        calls["n"] += 1
+        if calls["n"] < 3:
+            raise urlerror.URLError("connection reset")
+        class _R:
+            def __enter__(self): return self
+            def __exit__(self, *a): return False
+            def read(self): return b'{"choices":[{"message":{"tool_calls":[]}}]}'
+        return _R()
+
+    monkeypatch.setattr(mod.urlrequest, "urlopen", flaky)
+    monkeypatch.setattr(mod.time, "sleep", lambda s: None)
+    cfg = AgentConfig(retries=3)
+    body, _ = mod.chat(cfg, [{"role": "user", "content": "x"}], think=False)
+    assert calls["n"] == 3 and "choices" in body
+
+
+def test_chat_gives_up_with_a_clear_error(monkeypatch):
+    import agent as mod
+    from urllib import error as urlerror
+
+    monkeypatch.setattr(mod.urlrequest, "urlopen",
+                        lambda *a, **k: (_ for _ in ()).throw(urlerror.URLError("down")))
+    monkeypatch.setattr(mod.time, "sleep", lambda s: None)
+    with pytest.raises(mod.ChatError, match="after 2 attempts"):
+        mod.chat(AgentConfig(retries=2), [{"role": "user", "content": "x"}], think=False)
+
+
+# --------------------------------------------------- oracles that were wrong
+
+def test_alarm_task_needs_a_saved_alarm_not_a_typed_digit():
+    """This oracle used to pass on any typed string containing "7", so typing
+    `7` and stopping scored a success on "Set an alarm for 7:30 AM"."""
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["add_alarm"]
+
+    typed_only = task.build()
+    typed_only.launch("Clock")
+    typed_only.tap(540, 200)      # Alarm tab
+    typed_only.tap(540, 360)      # Add alarm
+    typed_only.tap(540, 360)
+    typed_only.input_text("7")
+    assert task.score(typed_only, declared_done=True)["success"] is False
+
+    both_fields = task.build()
+    both_fields.launch("Clock")
+    both_fields.tap(540, 200)
+    both_fields.tap(540, 360)
+    both_fields.tap(540, 360)
+    both_fields.input_text("7")
+    both_fields.tap(540, 520)
+    both_fields.input_text("30")
+    assert task.score(both_fields, declared_done=True)["success"] is False, \
+        "typing both fields without saving is not a set alarm"
+
+    both_fields.tap(540, 840)     # Save
+    assert task.score(both_fields, declared_done=True)["success"] is True
+
+
+def test_alarm_task_rejects_the_wrong_meridiem():
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["add_alarm"]
+    d = task.build()
+    d.launch("Clock"); d.tap(540, 200); d.tap(540, 360)
+    d.tap(540, 360); d.input_text("7")
+    d.tap(540, 520); d.input_text("30")
+    d.tap(540, 680)               # AM -> PM
+    d.tap(540, 840)               # Save
+    score = task.score(d, declared_done=True)
+    assert score["success"] is False and score["collateral_damage"] is True
+
+
+def test_a_saved_alarm_is_visible_on_the_screen():
+    """The oracle reads the clock app's list, so it works on a real phone."""
+    from phone_state import read_state
+    from tasks import SUITE_BY_ID
+
+    d = SUITE_BY_ID["add_alarm"].build()
+    d.launch("Clock"); d.tap(540, 200); d.tap(540, 360)
+    d.tap(540, 360); d.input_text("7")
+    d.tap(540, 520); d.input_text("30")
+    d.tap(540, 840)
+    assert read_state(d).has_text("7:30 AM")
+
+
+# ------------------------------------------------------------------ variants
+
+def test_every_task_has_variants_so_repeats_are_trials():
+    """Five runs of one fixture at temperature 0 measure repeatability. The
+    suite has to vary something for a repeat to be a trial."""
+    from tasks import SUITE
+
+    for task in SUITE:
+        assert len(task.variants) >= 1
+        ids = [v.variant_id for v in task.variants]
+        assert len(ids) == len(set(ids)), task.task_id
+    assert any(len(t.variants) > 1 for t in SUITE)
+
+
+def test_variants_change_the_phone_but_not_the_goal():
+    """A renamed or reordered control is still the same task."""
+    from phone_state import read_state
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on"]
+    labels_seen = set()
+    for run in range(len(task.variants)):
+        d = task.build(run)
+        d.launch("Settings")
+        d.tap(540, 360)
+        labels_seen.add(read_state(d).source)
+        # The goal predicate is unchanged; only the route differs.
+        assert task.score(d, declared_done=True)["success"] is False
+        d.state["wifi"] = True
+        assert task.score(d, declared_done=True)["success"] is True
+    assert labels_seen == {"sim"}
+
+
+def test_wlan_variant_renames_the_control_the_agent_must_find():
+    from phone_state import read_state
+    from tasks import SUITE_BY_ID
+
+    task = SUITE_BY_ID["wifi_on"]
+    variant = next(i for i, v in enumerate(task.variants) if v.variant_id == "wlan")
+    d = task.build(variant)
+    d.launch("Settings"); d.tap(540, 360)
+    assert read_state(d).has_text("WLAN")
+    assert not read_state(d).has_text("Wi-Fi")
+
+
+# -------------------------------------------------- real device honesty
+
+def test_history_dependent_task_refuses_to_score_on_a_real_phone():
+    """A phone cannot report which switches were flipped, so the task says so
+    instead of scoring from final state and calling a round trip a pass."""
+    from phone_state import PhoneState
+    from tasks import SUITE_BY_ID
+
+    state = PhoneState(source="device", settings={"wifi": True})
+    score = SUITE_BY_ID["wifi_on_already"].score(state, declared_done=True)
+    assert score["scorable"] is False
+    assert "simulator only" in score["reason"]
+    assert score["success"] is False
+
+
+def test_tasks_runnable_on_a_phone_declare_how_to_reset_it():
+    """Without a reset the run is scored against an unknown starting state."""
+    from tasks import SUITE_BY_ID
+
+    assert SUITE_BY_ID["wifi_on"].reset_adb
+    assert SUITE_BY_ID["wifi_off"].reset_adb
+    # and the sim-only one is not pretending otherwise
+    assert SUITE_BY_ID["wifi_on_already"].needs == ("toggled",)

benchmarks/edge_harness/android_agent/tool_parse.py

@@ -0,0 +1,155 @@
+"""Recover a tool call from what the model actually emitted.
+
+vLLM's stock `glm45` parser handles Spark's tool output most of the time --
+4 of 5 screens in a spot check -- and then does not. On the Wi-Fi toggle
+screen, the one where a switch's state decides the right action, the model
+dropped out of GLM-4.5 format entirely and emitted
+
+    [tap]
+    {"tool": "tap", "parameters": {"index": 1}
+
+with the JSON unterminated. `tool_calls` comes back empty, and an agent that
+only reads `tool_calls` sees a step with no action at all.
+
+For a phone agent that is the difference between finishing a task and
+stopping in the middle of it, so this module tries, in order:
+
+1. the parsed `tool_calls` (the normal path -- nothing here second-guesses it),
+2. a GLM-4.5 `<tool_call>` block left in the content,
+3. the bracketed `[name]` + JSON form above, repairing unbalanced braces,
+4. a bare JSON object naming a tool.
+
+Every recovery is reported, because an agent that quietly repairs its model's
+output will hide a regression in that model.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+
+_BRACKET = re.compile(r"\[([a-z_][a-z0-9_]*)\]", re.I)
+_GLM_CALL = re.compile(
+    r"<tool_call>\s*([a-z_][a-z0-9_]*)(.*?)</tool_call>", re.I | re.S)
+_GLM_ARG = re.compile(
+    r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>", re.I | re.S)
+
+_ARG_CONTAINERS = ("parameters", "arguments", "args")
+_NAME_KEYS = ("tool", "name", "tool_name", "function")
+
+
+@dataclass
+class ParsedCall:
+    name: str | None
+    args: dict
+    source: str
+    """Which path produced it: tool_calls | glm_block | bracketed | bare_json."""
+
+    @property
+    def recovered(self) -> bool:
+        return self.name is not None and self.source != "tool_calls"
+
+
+def _coerce(value: str):
+    """Tool arguments arrive as strings; indices must become integers."""
+    text = value.strip()
+    if re.fullmatch(r"-?\d+", text):
+        return int(text)
+    if text.lower() in ("true", "false"):
+        return text.lower() == "true"
+    return text
+
+
+def _balance(text: str) -> str:
+    """Close a JSON object the model stopped short of finishing."""
+    depth_curly = text.count("{") - text.count("}")
+    depth_square = text.count("[") - text.count("]")
+    if depth_curly < 0 or depth_square < 0:
+        return text
+    return text + "]" * depth_square + "}" * depth_curly
+
+
+def _load(text: str) -> dict | None:
+    for candidate in (text, _balance(text)):
+        try:
+            value = json.loads(candidate)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(value, dict):
+            return value
+    return None
+
+
+def _unwrap(payload: dict, fallback_name: str | None) -> tuple[str | None, dict]:
+    """Pull a (name, args) pair out of assorted self-describing shapes."""
+    name = fallback_name
+    for key in _NAME_KEYS:
+        value = payload.get(key)
+        if isinstance(value, str) and value:
+            name = value
+            break
+        if isinstance(value, dict) and isinstance(value.get("name"), str):
+            name = value["name"]
+            payload = {**payload, **value}
+            break
+    args: dict = {}
+    for key in _ARG_CONTAINERS:
+        inner = payload.get(key)
+        if isinstance(inner, dict):
+            args = dict(inner)
+            break
+        if isinstance(inner, str):
+            parsed = _load(inner)
+            if parsed is not None:
+                args = parsed
+                break
+    if not args:
+        args = {k: v for k, v in payload.items()
+                if k not in _NAME_KEYS + _ARG_CONTAINERS}
+    return name, args
+
+
+def _first_json_object(text: str) -> str | None:
+    start = text.find("{")
+    return text[start:] if start >= 0 else None
+
+
+def extract_tool_call(message: dict) -> ParsedCall:
+    """The one call to act on, from whichever shape the model used."""
+    calls = message.get("tool_calls") or []
+    if calls:
+        fn = calls[0].get("function", {}) or {}
+        raw = fn.get("arguments")
+        if isinstance(raw, dict):
+            args = raw
+        else:
+            args = _load(raw or "{}") or {}
+        if set(args) & set(_ARG_CONTAINERS):
+            _, args = _unwrap(args, fn.get("name"))
+        return ParsedCall(fn.get("name"), args, "tool_calls")
+
+    content = message.get("content") or ""
+    reasoning = message.get("reasoning_content") or ""
+    for text in (content, reasoning):
+        if not text.strip():
+            continue
+
+        if m := _GLM_CALL.search(text):
+            args = {k.strip(): _coerce(v) for k, v in _GLM_ARG.findall(m.group(2))}
+            return ParsedCall(m.group(1), args, "glm_block")
+
+        bracket = _BRACKET.search(text)
+        blob = _first_json_object(text)
+        if blob is not None:
+            payload = _load(blob)
+            if payload is not None:
+                name, args = _unwrap(payload, bracket.group(1) if bracket else None)
+                if name:
+                    return ParsedCall(name, args, "bracketed" if bracket else "bare_json")
+
+        if bracket and not blob:
+            # A naked "[press_key]" with no arguments is still an instruction.
+            return ParsedCall(bracket.group(1), {}, "bracketed")
+
+    return ParsedCall(None, {}, "none")

benchmarks/edge_harness/android_agent/ui_tree.py

@@ -0,0 +1,146 @@
+"""Turn a uiautomator dump into what a text-only agent can act on.
+
+Spark-X2.5 never sees a screenshot, so the whole agent hinges on this file:
+a screen has to arrive as a short, stable, indexed list of elements, and an
+index the model returns has to map back to a point on the glass.
+
+Two decisions shape the format, both forced by the model rather than by taste:
+
+* **Filter hard.** A real settings screen dumps 150+ nodes, most of them
+  layout containers. At 56 KiB/token of KV cache and a 1.7B model that both
+  costs context and buries the answer, so only nodes that are interactive or
+  carry text survive.
+* **Index, do not coordinate.** The model is asked for `index: 3`, never for
+  pixels -- a text model has no way to know where 3 is on the glass, and
+  coordinates in the transcript would be a source of silent nonsense. The
+  mapping back to a centre point happens here, where the bounds are known.
+
+The rendered form is the one the scenario fixtures were written in, so the
+decision benchmark and the live agent observe identically.
+"""
+
+from __future__ import annotations
+
+import re
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+
+_BOUNDS = re.compile(r"\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]")
+
+# Attributes worth showing when they are true, in the order a reader wants
+# them. `checked` earns its place: the one clearly wrong action in the earlier
+# measurements was tapping a Wi-Fi switch that was already on, which is only
+# avoidable if the state is on the screen the model sees.
+_FLAGS = ("clickable", "checked", "selected", "scrollable", "focused")
+
+
+@dataclass
+class Element:
+    index: int
+    cls: str
+    text: str
+    resource_id: str = ""
+    package: str = ""
+    bounds: tuple[int, int, int, int] = (0, 0, 0, 0)
+    flags: dict[str, bool] = field(default_factory=dict)
+    editable: bool = False
+
+    @property
+    def center(self) -> tuple[int, int]:
+        x1, y1, x2, y2 = self.bounds
+        return (x1 + x2) // 2, (y1 + y2) // 2
+
+    @property
+    def short_class(self) -> str:
+        return self.cls.rsplit(".", 1)[-1] if self.cls else "View"
+
+    def render(self) -> str:
+        parts = [f"[{self.index}]", self.short_class, f'"{self.text}"']
+        for flag in _FLAGS:
+            if flag in self.flags:
+                parts.append(f"{flag}={str(self.flags[flag]).lower()}")
+        return " ".join(parts)
+
+
+def _truthy(node: ET.Element, name: str) -> bool:
+    return node.get(name, "false") == "true"
+
+
+def _label(node: ET.Element) -> str:
+    """What to call this element, preferring what a person would read."""
+    for attr in ("text", "content-desc"):
+        value = (node.get(attr) or "").strip()
+        if value:
+            return value.replace("\n", " ")
+    # A bare resource id is a poor label but far better than an empty string
+    # when the element is the only way forward (unlabelled icon buttons).
+    rid = node.get("resource-id") or ""
+    return rid.rsplit("/", 1)[-1] if rid else ""
+
+
+def _interesting(node: ET.Element) -> bool:
+    if any(_truthy(node, a) for a in ("clickable", "checkable", "scrollable",
+                                      "long-clickable", "focusable")):
+        return True
+    return bool(_label(node)) and not _truthy(node, "password")
+
+
+def parse_bounds(raw: str | None) -> tuple[int, int, int, int]:
+    m = _BOUNDS.search(raw or "")
+    if not m:
+        return (0, 0, 0, 0)
+    return tuple(int(g) for g in m.groups())  # type: ignore[return-value]
+
+
+def parse_ui_dump(xml_text: str, *, max_elements: int = 60) -> list[Element]:
+    """uiautomator XML -> the elements worth showing, in reading order."""
+    root = ET.fromstring(xml_text)
+    elements: list[Element] = []
+    for node in root.iter("node"):
+        if not _interesting(node):
+            continue
+        label = _label(node)
+        clickable = _truthy(node, "clickable")
+        # An unlabelled, non-interactive node tells the model nothing it can
+        # use and costs it context.
+        if not label and not clickable and not _truthy(node, "checkable"):
+            continue
+        flags = {"clickable": clickable}
+        if _truthy(node, "checkable"):
+            flags["checked"] = _truthy(node, "checked")
+        if _truthy(node, "selected"):
+            flags["selected"] = True
+        if _truthy(node, "scrollable"):
+            flags["scrollable"] = True
+        if _truthy(node, "focused"):
+            flags["focused"] = True
+        elements.append(
+            Element(
+                index=len(elements),
+                cls=node.get("class", ""),
+                text=label,
+                resource_id=node.get("resource-id", ""),
+                package=node.get("package", ""),
+                bounds=parse_bounds(node.get("bounds")),
+                flags=flags,
+                editable="EditText" in (node.get("class") or ""),
+            )
+        )
+        if len(elements) >= max_elements:
+            break
+    return elements
+
+
+def render_tree(elements: list[Element]) -> str:
+    """The exact text the model is shown."""
+    return "\n".join(e.render() for e in elements)
+
+
+def screen_signature(elements: list[Element]) -> str:
+    """A cheap identity for a screen, to notice when an action changed nothing.
+
+    Labels and flags only: coordinates shift with animations and scroll offsets,
+    and treating those as a new screen would hide a stuck agent.
+    """
+    return "|".join(f"{e.short_class}:{e.text}:{e.flags.get('checked', '')}"
+                    for e in elements)

benchmarks/edge_harness/archive_profiles.py

@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Archive profiling evidence with byte hashes and reversible large-file compression.
+
+Run only after all writers exit. No benchmark result is changed by this utility.
+"""
+
+import argparse
+import gzip
+import hashlib
+import json
+import shutil
+from pathlib import Path
+
+
+def digest(path, compressed=False):
+    hasher = hashlib.sha256()
+    size = 0
+    opener = gzip.open if compressed else open
+    with opener(path, "rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            hasher.update(chunk)
+            size += len(chunk)
+    return hasher.hexdigest(), size
+
+
+def verify(root):
+    manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8"))
+    for record in manifest["files"]:
+        path = root / record["stored_path"]
+        assert path.resolve().is_relative_to(root.resolve()), "Archive path escapes root"
+        assert digest(path) == (record["stored_sha256"], record["stored_bytes"]), path
+        assert digest(path, record["encoding"] == "gzip") == (record["sha256"], record["bytes"]), path
+    return len(manifest["files"])
+
+
+def archive(source, destination):
+    assert source.resolve() != destination.resolve()
+    assert not destination.resolve().is_relative_to(source.resolve())
+    destination.mkdir(parents=True, exist_ok=False)
+    records = []
+    for path in sorted(source.rglob("*")):
+        relative = path.relative_to(source)
+        if not path.is_file() or "__pycache__" in relative.parts or path.suffix in (".pyc", ".tmp"):
+            continue
+        assert not path.is_symlink(), f"Unexpected evidence symlink: {relative}"
+        original_hash, original_size = digest(path)
+        compress = original_size > 50 * 1024 * 1024 and path.suffix != ".gz"
+        stored = Path("evidence") / relative
+        if compress:
+            stored = stored.with_name(stored.name + ".archive.gz")
+        target = destination / stored
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if compress:
+            with path.open("rb") as reader, target.open("wb") as writer:
+                with gzip.GzipFile(filename="", mode="wb", fileobj=writer, mtime=0) as packed:
+                    shutil.copyfileobj(reader, packed, length=1024 * 1024)
+        else:
+            shutil.copyfile(path, target)
+        stored_hash, stored_size = digest(target)
+        assert digest(path) == (original_hash, original_size), f"Source changed during archive: {relative}"
+        records.append(
+            {
+                "source_path": relative.as_posix(),
+                "stored_path": stored.as_posix(),
+                "sha256": original_hash,
+                "bytes": original_size,
+                "stored_sha256": stored_hash,
+                "stored_bytes": stored_size,
+                "encoding": "gzip" if compress else "identity",
+            }
+        )
+    (destination / "manifest.json").write_text(
+        json.dumps(
+            {
+                "schema": 1,
+                "scope": "Raw evidence bytes; hashes establish integrity, not model correctness or qualification.",
+                "files": records,
+            },
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    (destination / ".gitattributes").write_text("evidence/** -text -whitespace\n", encoding="utf-8")
+    return verify(destination)
+
+
+def restore(root, destination):
+    verify(root)
+    destination.mkdir(parents=True, exist_ok=False)
+    manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8"))
+    for record in manifest["files"]:
+        target = destination / record["source_path"]
+        assert target.resolve().is_relative_to(destination.resolve()), "Restore path escapes root"
+        target.parent.mkdir(parents=True, exist_ok=True)
+        opener = gzip.open if record["encoding"] == "gzip" else open
+        with opener(root / record["stored_path"], "rb") as reader, target.open("wb") as writer:
+            shutil.copyfileobj(reader, writer, length=1024 * 1024)
+        assert digest(target) == (record["sha256"], record["bytes"])
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    sub = parser.add_subparsers(dest="operation", required=True)
+    create = sub.add_parser("create")
+    create.add_argument("source", type=Path)
+    create.add_argument("destination", type=Path)
+    check = sub.add_parser("verify")
+    check.add_argument("archive", type=Path)
+    unpack = sub.add_parser("restore")
+    unpack.add_argument("archive", type=Path)
+    unpack.add_argument("destination", type=Path)
+    args = parser.parse_args()
+    if args.operation == "create":
+        print(f"Archived and verified {archive(args.source, args.destination)} files")
+    elif args.operation == "verify":
+        print(f"Verified {verify(args.archive)} files")
+    else:
+        restore(args.archive, args.destination)
+        print(f"Restored evidence to {args.destination}")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/bench_cpu_vllm.py

@@ -0,0 +1,106 @@
+"""pp512 / tg128 for vLLM on CPU, mirroring `llama-bench` semantics.
+
+pp512: time to process a 512-token prompt (prefill throughput).
+tg128: time to generate 128 tokens (decode throughput), prefill subtracted.
+"""
+import argparse, json, os, sys, time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from bench_harness import provenance  # noqa: E402
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", default="XHToken/Spark-X2.5-1.7B")
+    ap.add_argument("--dtype", default="bfloat16")
+    ap.add_argument("--pp", type=int, default=512)
+    ap.add_argument("--tg", type=int, default=128)
+    ap.add_argument("--reps", type=int, default=3)
+    ap.add_argument("--max-model-len", type=int, default=2048)
+    ap.add_argument("--enforce-eager", action="store_true")
+    ap.add_argument("--quantization", default=None)
+    ap.add_argument("--tag", default="vllm")
+    ap.add_argument("--no-detokenize", action="store_true",
+                    help="skip incremental detokenization, as llama-bench does")
+    ap.add_argument("--compilation-config", default=None,
+                    help="JSON passed straight to LLM(compilation_config=...)")
+    ap.add_argument("--out", default="vllm_cpu_bench.json")
+    a = ap.parse_args()
+
+    import vllm_omni  # noqa: F401
+    from vllm import LLM, SamplingParams
+
+    t0 = time.perf_counter()
+    # Prefix caching must be off: the pp measurement replays one prompt, and a
+    # cache hit would report a prefill that never happened. llama-bench has no
+    # such cache, so leaving it on would not be a comparison.
+    extra = {}
+    if a.compilation_config:
+        extra["compilation_config"] = json.loads(a.compilation_config)
+    llm = LLM(model=a.model, dtype=a.dtype, max_model_len=a.max_model_len,
+              enforce_eager=a.enforce_eager, max_num_seqs=1,
+              quantization=a.quantization, enable_prefix_caching=False,
+              trust_remote_code=False, disable_log_stats=True, **extra)
+    init_s = time.perf_counter() - t0
+
+    def rss_mb(field):
+        with open("/proc/self/status") as f:
+            for line in f:
+                if line.startswith(field):
+                    return round(int(line.split()[1]) / 1024, 1)
+        return None
+
+    prompt_pp = {"prompt_token_ids": list(range(10, 10 + a.pp))}
+    prompt_tg = {"prompt_token_ids": [10]}
+    det = not a.no_detokenize
+    sp_pre = SamplingParams(temperature=0.0, max_tokens=1, ignore_eos=True,
+                            detokenize=det)
+    sp_gen = SamplingParams(temperature=0.0, max_tokens=a.tg, ignore_eos=True,
+                            detokenize=det)
+
+    def timed(prompt, sp, reps):
+        best = []
+        for _ in range(reps):
+            s = time.perf_counter()
+            llm.generate([prompt], sp, use_tqdm=False)
+            best.append(time.perf_counter() - s)
+        return best
+
+    timed(prompt_pp, sp_pre, 1)  # warmup
+
+    pp_t = timed(prompt_pp, sp_pre, a.reps)
+    tg_t = timed(prompt_tg, sp_gen, a.reps)
+    tg1_t = timed(prompt_tg, sp_pre, a.reps)  # 1-token prefill overhead
+
+    pp_best, tg_best, tg1_best = min(pp_t), min(tg_t), min(tg1_t)
+    res = {
+        "tag": a.tag, "model": a.model, "dtype": a.dtype,
+        "compilation_config": a.compilation_config,
+        "detokenize": not a.no_detokenize,
+        "quantization": a.quantization,
+        "enforce_eager": a.enforce_eager, "init_s": round(init_s, 2),
+        # Which build, which threads, which commits -- a throughput number
+        # without these cannot be invalidated later, only doubted.
+        "provenance": provenance({
+            "model_path": os.path.realpath(a.model) if os.path.exists(a.model)
+            else a.model,
+            "pp": a.pp, "tg": a.tg, "reps": a.reps,
+            "max_model_len": a.max_model_len,
+        }),
+        "rss_mb": rss_mb("VmRSS:"), "rss_peak_mb": rss_mb("VmHWM:"),
+        "threads_bind": os.environ.get("VLLM_CPU_OMP_THREADS_BIND", ""),
+        "omp_num_threads": os.environ.get("OMP_NUM_THREADS", ""),
+        "pp_tokens": a.pp, "pp_s": round(pp_best, 4),
+        "pp_tps": round(a.pp / pp_best, 2),
+        "tg_tokens": a.tg,
+        "tg_s": round(tg_best - tg1_best, 4),
+        "tg_tps": round((a.tg - 1) / max(tg_best - tg1_best, 1e-9), 2),
+        "raw": {"pp": pp_t, "tg": tg_t, "tg1": tg1_t},
+    }
+    print(json.dumps({k: v for k, v in res.items() if k != "raw"}, indent=2))
+    with open(a.out, "a") as f:
+        f.write(json.dumps(res) + "\n")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/bench_harness.py

@@ -0,0 +1,634 @@
+"""Run benchmark arms so the numbers survive a shared machine.
+
+Every wrong conclusion this study produced came from measurement, not from
+code: a sequential grid that swung the llama.cpp control 86.7 -> 60.4 tok/s
+mid-run, a 3.1 GB "saving" that was really `VmRSS` counting the mmap'd
+checkpoint's pages, and a W4A8/W4A16 ordering that flipped between grids. The
+discipline that fixed each of them is the same, so it lives here instead of
+being re-typed into the next shell script:
+
+* **Rotate the arm order every pass.** Drift hits arm 1 and arm 5 differently,
+  and interleaving alone does not fix it: if every pass runs the arms in the
+  same order, arm 1 is always the one measured first and carries whatever the
+  machine was doing at the start of each pass. Pass ``p`` starts at arm
+  ``(p - 1) mod n``.
+* **Repeat, and report the spread.** A single number from a shared host is a
+  claim about the host, not the code. An arm whose passes disagree by more
+  than `--noise-threshold` is flagged, and the summary says so.
+* **Record how busy the machine was, and refuse when it is.** A pass taken
+  under load is kept and labelled rather than silently averaged in -- but load
+  average is a whole-host number, and this host has 224 cores, so it says
+  nothing about the 24 this benchmark is pinned to. The pre-flight reads
+  per-core utilisation for the *target* cores, and a host lock keeps two
+  benchmarks off the same cores in the first place. Two sessions benchmarking
+  at once wrecked several measurements here, and one grid's control drifted
+  30% mid-run.
+* **Prefer allocation over memory.** `RssAnon` repeats to within 0.5 MB where
+  `VmRSS` moved 1-2 GB between identical runs.
+* **Know which direction is better.** ``best`` is the maximum for a rate and
+  the minimum for memory, init time or latency. Reporting the maximum for every
+  metric published the *worst* pass as the headline memory figure.
+* **Stamp provenance.** A number whose model, build and thread set were not
+  recorded cannot be retracted later, only doubted -- and this study has
+  retracted four.
+* **Give an arm its own compile cache when it changes the graph.** vLLM keys
+  its AOT compile cache on the *config*, so two arms that differ only by an
+  environment variable share a cached artifact -- and if that variable changes
+  the traced graph, later passes load the wrong one. Three passes of a prefill
+  grid died with ``KeyError: 'weight_halves'`` inside an AOT-loaded forward
+  after the first pass saved an artifact for the other path. Set
+  ``VLLM_CACHE_ROOT`` per arm, and note that the first pass of each arm then
+  pays a cold compile, which is why ``init_s`` is not comparable across passes.
+* **Gate on what the host can actually resolve.** ``--check`` compares against
+  a checked-in expectations file, with a band per metric taken from that
+  metric's *observed* spread: 0.2% for ``RssAnon``, ~5% for per-op self time,
+  and nothing for end-to-end throughput, which swings ~30% here and cannot
+  carry a gate at all. A band is an argument about what is measurable, not a
+  tolerance someone picked.
+
+The arms are subprocess invocations of the existing single-run benchmarks, so
+the measurement code stays the one that was verified; this only decides what
+to run, when, and how to read it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+import os
+import platform
+import shlex
+import statistics
+import subprocess
+import time
+from dataclasses import dataclass, field
+
+
+def load_average() -> float:
+    return os.getloadavg()[0]
+
+
+def parse_cores(spec: str) -> list[int]:
+    """'0-23,48-71' -> [0..23, 48..71]"""
+    cores: list[int] = []
+    for part in spec.split(","):
+        part = part.strip()
+        if not part:
+            continue
+        if "-" in part:
+            lo, hi = part.split("-")
+            cores.extend(range(int(lo), int(hi) + 1))
+        else:
+            cores.append(int(part))
+    return sorted(set(cores))
+
+
+@dataclass
+class Arm:
+    tag: str
+    command: list[str]
+    env: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class Observation:
+    tag: str
+    pass_no: int
+    metrics: dict[str, float]
+    load_before: float
+    load_after: float
+    wall_s: float
+    ok: bool
+    error: str = ""
+
+    @property
+    def busy(self) -> bool:
+        return max(self.load_before, self.load_after) > BUSY_LOAD
+
+
+BUSY_LOAD = 8.0
+"""Above this 1-minute load average a pass is labelled contended.
+
+A poor default on its own: load average is whole-host, and on a 224-core
+machine a 24-thread benchmark contributes ~24 to it by itself, so a fixed 8
+labels every pass contended and the label stops meaning anything. ``main()``
+derives it from the target core count instead -- our own threads plus slack --
+and the per-core pre-flight is the guard that actually decides whether to run.
+"""
+
+LOWER_IS_BETTER = (
+    "rss", "mem", "mb", "init", "load_s", "latency", "ms", "_s", "time", "peak",
+)
+"""Substrings that mark a metric where smaller wins. Rates (`tg_tps`, `pp_tps`)
+take the maximum; memory, init time and latency take the minimum."""
+
+
+HOST_LOCK_PATH = os.environ.get("SPARK_EDGE_LOCK", "/tmp/spark_edge_bench.lock")
+"""Host-level lock. It has to live somewhere every session can see, which a
+session-private scratchpad by definition is not -- that is the whole point of
+a host lock. Override with ``SPARK_EDGE_LOCK``."""
+
+CORE_BUSY_PCT = 50.0
+"""A target core sustained above this is a straggler. OpenMP barriers mean the
+slowest thread gates the whole region, so one heavily-loaded core costs far
+more than its share -- which is why this is a separate test from the aggregate
+below rather than a single averaged number."""
+
+STOLEN_PCT = 5.0
+"""Refuse when this much of the target cores' aggregate capacity is going to
+someone else, which bounds the bias on the measurement.
+
+A flat per-core threshold does not work on this host. With 224 cores and
+several users there is always a transient blip somewhere: a 20% rule over a
+single 0.5 s window refused three consecutive launches on cores that were
+idle seconds earlier, which makes the guard useless rather than strict. So
+utilisation is sampled twice and the per-core minimum is taken -- a spike in
+one window does not count as load -- and the decision is made on aggregate
+stolen capacity plus the straggler rule."""
+
+
+def _cpu_times() -> dict[int, tuple[int, int]]:
+    """(busy, total) jiffies per core, from /proc/stat."""
+    out: dict[int, tuple[int, int]] = {}
+    with open("/proc/stat") as f:
+        for line in f:
+            if not line.startswith("cpu") or line.startswith("cpu "):
+                continue
+            parts = line.split()
+            try:
+                cpu = int(parts[0][3:])
+            except ValueError:
+                continue
+            values = [int(v) for v in parts[1:11]]
+            idle = values[3] + values[4]           # idle + iowait
+            total = sum(values)
+            out[cpu] = (total - idle, total)
+    return out
+
+
+def core_utilization(cores: list[int], window_s: float = 0.5) -> dict[int, float]:
+    """Percentage busy per core over ``window_s``."""
+    before = _cpu_times()
+    time.sleep(window_s)
+    after = _cpu_times()
+    usage: dict[int, float] = {}
+    for cpu in cores:
+        if cpu not in before or cpu not in after:
+            continue
+        busy = after[cpu][0] - before[cpu][0]
+        total = after[cpu][1] - before[cpu][1]
+        usage[cpu] = 0.0 if total <= 0 else busy / total * 100.0
+    return usage
+
+
+def preflight(cores: list[int], *, max_busy_pct: float = CORE_BUSY_PCT,
+              max_stolen_pct: float = STOLEN_PCT,
+              window_s: float = 0.75, samples: int = 2) -> dict:
+    """Refuse to start when contention would bias the measurement.
+
+    Two independent windows; a core's load is the *minimum* of them, so a
+    transient spike is not mistaken for a busy core. Then two tests: aggregate
+    stolen capacity (bounds the average bias) and a per-core straggler rule
+    (one slow core gates every OpenMP barrier).
+    """
+    windows = [core_utilization(cores, window_s) for _ in range(max(samples, 1))]
+    usage = {c: min(w.get(c, 0.0) for w in windows) for c in cores
+             if any(c in w for w in windows)}
+    stolen = (sum(usage.values()) / len(usage)) if usage else 0.0
+    stragglers = {c: round(u, 1) for c, u in usage.items() if u > max_busy_pct}
+    return {"cores": cores,
+            "max_busy_pct": max_busy_pct, "max_stolen_pct": max_stolen_pct,
+            "utilization": {c: round(u, 1) for c, u in usage.items()},
+            "stolen_pct": round(stolen, 2),
+            "busy_cores": stragglers,
+            "clear": stolen <= max_stolen_pct and not stragglers}
+
+
+class HostLock:
+    """An advisory lock so two sessions do not benchmark the same cores.
+
+    Cooperative, and says so: it excludes other runs that take it, not the
+    unrelated job someone starts by hand. That is what the pre-flight is for.
+    """
+
+    def __init__(self, path: str = HOST_LOCK_PATH, note: str = ""):
+        self.path, self.note, self._fd = path, note, None
+
+    def __enter__(self) -> "HostLock":
+        import fcntl
+
+        self._fd = open(self.path, "a+")
+        try:
+            fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            self._fd.seek(0)
+            holder = self._fd.read().strip()[-200:]
+            self._fd.close()
+            self._fd = None
+            raise SystemExit(
+                f"another benchmark holds {self.path}: {holder or 'unknown'}. "
+                f"Wait for it, or set SPARK_EDGE_LOCK to a different path if "
+                f"you really are on disjoint cores."
+            )
+        self._fd.seek(0)
+        self._fd.truncate()
+        self._fd.write(f"pid={os.getpid()} {time.strftime('%H:%M:%S')} {self.note}\n")
+        self._fd.flush()
+        return self
+
+    def __exit__(self, *exc) -> None:
+        if self._fd is not None:
+            import fcntl
+
+            fcntl.flock(self._fd, fcntl.LOCK_UN)
+            self._fd.close()
+            self._fd = None
+
+
+DEFAULT_BANDS_PCT = {
+    "rss_anon_mb": 1.5,
+    "peak_rss_anon_mb": 2.0,
+    "rss_anon_at_context_mb": 2.0,
+    "init_s": 12.0,
+    "ready_s": 12.0,
+    "import_s": 12.0,
+    "step_ms": 6.0,
+    "gemm_ms": 6.0,
+}
+"""Regression bands, per metric, from the spread each one actually shows on
+this host. ``RssAnon`` repeats to 0.2% and is given 1.5% for safety; init time
+is dominated by page cache and repeats far less tightly; per-op self time
+reproduces to ~5%.
+
+Throughput is deliberately absent. ``tg_tps`` swings ~30% run to run on this
+shared host, so any band wide enough not to fire constantly is too wide to
+catch a real regression. Gate on memory and per-op time instead; that is the
+whole reason the per-op profiler exists."""
+
+UNGATEABLE = {"tg_tps", "pp_tps", "tps"}
+"""Metrics this host cannot resolve well enough to gate on."""
+
+
+def check_expectations(summaries: list["ArmSummary"], expected: dict) -> list[dict]:
+    """Compare measured medians against a checked-in expectations file.
+
+    The median, not the best: `best` is one pass and a gate that fires on the
+    luckiest run of a noisy metric is a coin toss.
+    """
+    bands = {**DEFAULT_BANDS_PCT, **(expected.get("bands_pct") or {})}
+    arms = expected.get("arms") or {}
+    results = []
+    for s in summaries:
+        want = (arms.get(s.tag) or {}).get(s.metric)
+        if want is None:
+            results.append({"arm": s.tag, "metric": s.metric, "status": "unlisted",
+                            "measured": s.median, "expected": None})
+            continue
+        if s.metric in UNGATEABLE:
+            results.append({"arm": s.tag, "metric": s.metric, "status": "ungateable",
+                            "measured": s.median, "expected": want,
+                            "why": "this host cannot resolve it"})
+            continue
+        band = bands.get(s.metric, 5.0)
+        drift = abs(s.median - want) / want * 100.0 if want else 0.0
+        worse = (s.median > want) if s.lower_is_better else (s.median < want)
+        results.append({
+            "arm": s.tag, "metric": s.metric,
+            "measured": s.median, "expected": want,
+            "drift_pct": round(drift, 2), "band_pct": band,
+            "status": ("ok" if drift <= band else
+                       "REGRESSION" if worse else "improved"),
+        })
+    return results
+
+
+def format_check(results: list[dict]) -> str:
+    if not results:
+        return "nothing to check"
+    width = max(len(r["arm"]) for r in results)
+    lines = []
+    for r in results:
+        if r["status"] in ("unlisted", "ungateable"):
+            lines.append(f"  {r['arm'].ljust(width)}  {r['metric']:22s} "
+                         f"{r['measured']:9.2f}  {r['status']}"
+                         + (f" ({r['why']})" if r.get("why") else ""))
+        else:
+            lines.append(f"  {r['arm'].ljust(width)}  {r['metric']:22s} "
+                         f"{r['measured']:9.2f}  vs {r['expected']:9.2f}  "
+                         f"{r['drift_pct']:+5.2f}% (band {r['band_pct']:.1f}%)  "
+                         f"{r['status']}")
+    return "\n".join(lines)
+
+
+def metric_is_lower_better(metric: str) -> bool:
+    name = metric.lower()
+    if name.endswith(("_tps", "_tok_s", "tps")):
+        return False
+    return any(tok in name for tok in LOWER_IS_BETTER)
+
+
+def provenance(extra: dict | None = None) -> dict:
+    """Everything needed to decide later whether a number is still valid.
+
+    Saved profiles in this tree carry no model path, context length or build,
+    so two of them are indistinguishable from each other and one set silently
+    describes a checkpoint that was later found wrong. This is the fix.
+    """
+    root = os.environ.get("EMBEDDING_INFER_ROOT", "/data/zhoutaichang/embedding_infer")
+
+    def _sha(rel: str) -> str | None:
+        try:
+            out = subprocess.run(["git", "-C", os.path.join(root, rel),
+                                  "rev-parse", "HEAD"],
+                                 capture_output=True, text=True, timeout=10)
+            return out.stdout.strip() or None
+        except Exception:
+            return None
+
+    try:
+        affinity = sorted(os.sched_getaffinity(0))
+    except AttributeError:
+        affinity = []
+    return {
+        "recorded_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+        "host": platform.node(),
+        "python": platform.python_version(),
+        "cpu_count": os.cpu_count(),
+        "n_affinity": len(affinity),
+        "affinity": affinity,
+        "commits": {
+            "embedding_infer": _sha("."),
+            "vllm_omni": _sha("vllm-omni"),
+            "vllm": _sha("vllm"),
+        },
+        "env": {k: v for k, v in sorted(os.environ.items())
+                if k.startswith(("OMP_", "KMP_", "MKL_", "VLLM_", "LD_PRELOAD",
+                                 "CUDA_VISIBLE_DEVICES"))},
+        "loadavg": os.getloadavg(),
+        **(extra or {}),
+    }
+
+
+def run_once(arm: Arm, pass_no: int, timeout: float, keys: list[str]) -> Observation:
+    env = {**os.environ, **arm.env}
+    load_before = load_average()
+    t0 = time.perf_counter()
+    proc = subprocess.run(arm.command, env=env, capture_output=True,
+                          text=True, timeout=timeout)
+    wall = time.perf_counter() - t0
+    load_after = load_average()
+    if proc.returncode != 0:
+        return Observation(arm.tag, pass_no, {}, load_before, load_after, wall,
+                           ok=False, error=proc.stderr.strip()[-400:])
+    metrics = _scrape(proc.stdout, keys)
+    return Observation(arm.tag, pass_no, metrics, load_before, load_after, wall,
+                       ok=bool(metrics))
+
+
+def _scrape(stdout: str, keys: list[str]) -> dict[str, float]:
+    """Pull metrics out of the last JSON object the run printed."""
+    start = stdout.rfind("{")
+    while start >= 0:
+        try:
+            payload = json.loads(stdout[start:stdout.rfind("}") + 1])
+        except json.JSONDecodeError:
+            start = stdout.rfind("{", 0, start)
+            continue
+        return {k: float(payload[k]) for k in keys
+                if isinstance(payload.get(k), (int, float))}
+    return {}
+
+
+@dataclass
+class ArmSummary:
+    tag: str
+    metric: str
+    best: float
+    median: float
+    spread_pct: float
+    n: int
+    n_busy: int
+    lower_is_better: bool = False
+    noise_threshold: float = 8.0
+
+    @property
+    def noisy(self) -> bool:
+        return self.spread_pct > self.noise_threshold
+
+
+NOISE_THRESHOLD = 8.0
+
+
+def summarize(observations: list[Observation], metric: str,
+              noise_threshold: float = NOISE_THRESHOLD,
+              lower_is_better: bool | None = None) -> list[ArmSummary]:
+    """Collapse passes into one row per arm.
+
+    ``lower_is_better`` defaults to reading the metric name, so `rss_anon_mb`
+    and `init_s` take their minimum while `tg_tps` takes its maximum. Taking
+    the maximum for every metric is how the *worst* memory pass came to be
+    published as the best one.
+    """
+    if lower_is_better is None:
+        lower_is_better = metric_is_lower_better(metric)
+    by_tag: dict[str, list[Observation]] = {}
+    for obs in observations:
+        if obs.ok and metric in obs.metrics:
+            by_tag.setdefault(obs.tag, []).append(obs)
+    summaries = []
+    for tag, rows in by_tag.items():
+        values = [r.metrics[metric] for r in rows]
+        lo, hi = min(values), max(values)
+        spread = 0.0 if hi == 0 else (hi - lo) / hi * 100.0
+        summaries.append(ArmSummary(
+            tag=tag, metric=metric, best=lo if lower_is_better else hi,
+            median=statistics.median(values), spread_pct=spread,
+            n=len(values), n_busy=sum(1 for r in rows if r.busy),
+            lower_is_better=lower_is_better, noise_threshold=noise_threshold,
+        ))
+    return sorted(summaries, key=lambda s: s.best if lower_is_better else -s.best)
+
+
+def indistinguishable(summaries: list[ArmSummary]) -> str | None:
+    """Say so when the gap between the top two arms is inside the noise.
+
+    Per-op self time resolves ~5% on this host; end-to-end throughput resolves
+    ~30%. An 8% difference in tg_tps is not a result, and publishing it as one
+    is how the W4A8/W4A16 ordering came to flip between grids.
+    """
+    if len(summaries) < 2:
+        return None
+    first, second = summaries[0], summaries[1]
+    if not first.best:
+        return None
+    gap = abs(first.best - second.best) / abs(first.best) * 100.0
+    band = max(first.spread_pct, second.spread_pct,
+               DEFAULT_BANDS_PCT.get(first.metric, first.noise_threshold))
+    if gap > band:
+        return None
+    return (f"{first.tag} and {second.tag} differ by {gap:.1f}% on "
+            f"{first.metric}, inside this measurement's own spread "
+            f"({band:.1f}%). That is not a ranking. For a difference this "
+            f"size use per-op self time, which reproduces to ~5%: "
+            f"profile_cpu_step.py --model <build> --prompt-len <ctx>")
+
+
+def format_summary(summaries: list[ArmSummary]) -> str:
+    if not summaries:
+        return "no successful observations"
+    width = max(len(s.tag) for s in summaries)
+    direction = "min" if summaries[0].lower_is_better else "max"
+    lines = [f"{'arm'.ljust(width)}  {f'best({direction})':>9s} {'median':>9s} "
+             f"{'spread':>7s}  n  notes"]
+    for s in summaries:
+        notes = []
+        if s.noisy:
+            notes.append(f"NOISY (>{s.noise_threshold:.0f}%)")
+        if s.n_busy:
+            notes.append(f"{s.n_busy} pass(es) under load")
+        if s.n < 2:
+            notes.append("single pass — not a result")
+        lines.append(f"{s.tag.ljust(width)}  {s.best:9.2f} {s.median:9.2f} "
+                     f"{s.spread_pct:6.1f}%  {s.n}  {'; '.join(notes)}")
+    warning = indistinguishable(summaries)
+    if warning:
+        lines.append("")
+        lines.append(f"  NOT A RANKING: {warning}")
+    return "\n".join(lines)
+
+
+def run_grid(arms: list[Arm], *, passes: int, metric: str, timeout: float,
+             keys: list[str], settle_s: float = 0.0,
+             busy_load: float = BUSY_LOAD) -> list[Observation]:
+    """Run every arm each pass, rotating the starting arm between passes.
+
+    Interleaving alone leaves arm 0 permanently first, so whatever the machine
+    does at the start of a pass always lands on the same arm. Pass ``p`` starts
+    at arm ``(p - 1) mod n``.
+    """
+    globals()["BUSY_LOAD"] = busy_load
+    observations: list[Observation] = []
+    for p in range(1, passes + 1):
+        shift = (p - 1) % len(arms) if arms else 0
+        for arm in arms[shift:] + arms[:shift]:
+            if settle_s:
+                time.sleep(settle_s)
+            obs = run_once(arm, p, timeout, keys)
+            observations.append(obs)
+            status = "ok" if obs.ok else f"FAILED: {obs.error[:80]}"
+            value = obs.metrics.get(metric)
+            shown = f"{value:.2f}" if value is not None else "-"
+            flag = "  [busy]" if obs.busy else ""
+            print(f"  pass {p} {arm.tag:24s} {metric}={shown:>9s}  "
+                  f"load {obs.load_before:.1f}->{obs.load_after:.1f}{flag}  {status}",
+                  flush=True)
+    return observations
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--spec", required=True,
+                    help="JSON file: {\"arms\": [{\"tag\":..., \"command\":[...], \"env\":{...}}]}")
+    ap.add_argument("--passes", type=int, default=3)
+    ap.add_argument("--metric", default="tg_tps")
+    ap.add_argument("--keys", default="tg_tps,pp_tps,init_s,rss_anon_mb,rss_mb")
+    ap.add_argument("--timeout", type=float, default=2400)
+    ap.add_argument("--settle-s", type=float, default=2.0)
+    ap.add_argument("--busy-load", type=float, default=None,
+                    help="load average above which a pass is labelled "
+                         "contended; defaults to (target cores + 8), since "
+                         "the run itself contributes about one per core")
+    ap.add_argument("--noise-threshold", type=float, default=NOISE_THRESHOLD)
+    ap.add_argument("--out", default=None)
+    ap.add_argument("--cores", default="",
+                    help="target cores, e.g. '0-23'. Checked for other "
+                         "people's work before the run starts.")
+    ap.add_argument("--allow-busy", action="store_true",
+                    help="run even if the target cores are already loaded "
+                         "(the result is labelled, not trusted)")
+    ap.add_argument("--check", default=None,
+                    help="expectations JSON; exit non-zero on a regression "
+                         "outside the metric's measured band")
+    ap.add_argument("--no-lock", action="store_true",
+                    help="skip the host lock (only if you are on cores no "
+                         "other benchmark uses)")
+    a = ap.parse_args()
+
+    spec = json.loads(open(a.spec).read())
+    arms = [Arm(tag=x["tag"],
+                command=x["command"] if isinstance(x["command"], list)
+                else shlex.split(x["command"]),
+                env={k: str(v) for k, v in (x.get("env") or {}).items()})
+            for x in spec["arms"]]
+    keys = [k.strip() for k in a.keys.split(",") if k.strip()]
+
+    cores = parse_cores(a.cores) if a.cores else sorted(os.sched_getaffinity(0))
+    # Our own run puts ~len(cores) on the load average, so a threshold below
+    # that labels every pass contended and tells us nothing.
+    busy_load = a.busy_load if a.busy_load is not None else len(cores) + 8.0
+    check = preflight(cores)
+    if not check["clear"]:
+        detail = []
+        if check["stolen_pct"] > check["max_stolen_pct"]:
+            detail.append(f"{check['stolen_pct']}% of the cores' capacity is "
+                          f"someone else's (limit {check['max_stolen_pct']}%)")
+        if check["busy_cores"]:
+            detail.append("sustained stragglers: "
+                          + ", ".join(f"{c}={p}%"
+                                      for c, p in check["busy_cores"].items()))
+        msg = "contended target cores: " + "; ".join(detail)
+        if not a.allow_busy:
+            raise SystemExit(
+                msg + "\n  Another job is on these cores; this run would "
+                      "measure the contention, not the code. Wait, pick other "
+                      "cores with --cores, or pass --allow-busy to label the "
+                      "result instead of trusting it.")
+        print(f"WARNING: {msg} (--allow-busy)")
+
+    print(f"{len(arms)} arms x {a.passes} passes, rotating; metric={a.metric}; "
+          f"{len(cores)} cores, pre-flight "
+          f"{'clear' if check['clear'] else 'BUSY'} "
+          f"({check['stolen_pct']}% stolen), "
+          f"busy-label above load {busy_load:.0f}")
+
+    lock = contextlib.nullcontext() if a.no_lock else HostLock(note=a.metric)
+    with lock:
+        observations = run_grid(arms, passes=a.passes, metric=a.metric,
+                                timeout=a.timeout, keys=keys,
+                                settle_s=a.settle_s, busy_load=busy_load)
+    summaries = summarize(observations, a.metric, a.noise_threshold)
+    print("\n" + format_summary(summaries))
+
+    checks = []
+    if a.check:
+        checks = check_expectations(summaries, json.loads(open(a.check).read()))
+        print("\nregression check\n" + format_check(checks))
+
+    if a.out:
+        with open(a.out, "w") as f:
+            json.dump({
+                "metric": a.metric, "passes": a.passes,
+                "lower_is_better": metric_is_lower_better(a.metric),
+                "provenance": provenance({
+                    "arms": [{"tag": x.tag, "command": x.command, "env": x.env}
+                             for x in arms],
+                    "preflight": check,
+                    "busy_load_threshold": busy_load,
+                }),
+                "observations": [vars(o) for o in observations],
+                "summary": [vars(s) for s in summaries],
+                "checks": checks,
+            }, f, indent=2)
+        print(f"\nwrote {a.out}")
+
+    regressions = [c for c in checks if c["status"] == "REGRESSION"]
+    if regressions:
+        raise SystemExit(
+            f"{len(regressions)} regression(s): "
+            + ", ".join(f"{c['arm']}/{c['metric']} {c['drift_pct']:+.2f}%"
+                        for c in regressions))
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/bench_init_memory.py

@@ -0,0 +1,297 @@
+"""Init time and resident memory, which decide whether a model can be served
+from an edge box at all.
+
+Three quantities, because the plan's items are defined in terms of them and
+the earlier version of this file measured none of them:
+
+* **ready_s -- process start to serving.** The old ``init_s`` started its timer
+  *after* ``import vllm_omni``, so the 14.4 s of import cost the plan wants to
+  cut was invisible to the benchmark that was supposed to prove the cut. The
+  clock now starts at process creation, read from ``/proc/self/stat``, so
+  interpreter start-up and imports are inside it. ``init_s`` is still reported,
+  unchanged in meaning, so older rows stay comparable.
+* **peak_rss_anon_mb -- the high-water mark, sampled.** What an edge device has
+  to survive is the *peak*, not the steady state: this model loads at 7338 MB
+  and settles at 3510 MB only after the allocator is asked for its arenas back,
+  and a device that cannot hold the peak never reaches the release. There is no
+  kernel high-water mark for RssAnon (``VmHWM`` and ``ru_maxrss`` both track
+  VmRSS, which counts the mmap'd checkpoint and moved 1-2 GB between identical
+  runs), so a sampler thread takes it.
+* **rss_anon_at_context_mb -- memory once the cache is actually populated.**
+  The old default generated 8 tokens, so the KV cache was allocated and never
+  touched; "KV sizing saves only 188 MB" was measured there. ``--context-len``
+  fills it.
+
+And one correction that changes what the older rows mean. Everything here used
+to read ``/proc/self/status``, i.e. **this process only**. That is the right
+number only when the model is in-process, which happens when
+``VLLM_ENABLE_V1_MULTIPROCESSING=0`` keeps the executor at ``uni``
+(``vllm/platforms/cpu.py:273`` promotes it to ``mp`` otherwise). Every earlier
+run set that variable, so the published figures are in-process figures. Run the
+same benchmark at vLLM's defaults and the weights land in a worker process the
+benchmark never looked at: 796 MB "resident" for a model whose weights alone
+are 1.4 GB. Resident memory is now summed over the **process tree**, which is
+what an edge device actually has to hold, and both numbers are reported so the
+older rows remain comparable.
+
+Every row carries provenance, so it can be invalidated later instead of merely
+doubted.
+"""
+import time
+
+_T_IMPORT = time.time()
+
+import argparse, json, os, resource, sys, threading  # noqa: E402
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from bench_harness import provenance  # noqa: E402
+
+
+def _rss(field: str, pid: str | int = "self") -> float | None:
+    try:
+        with open(f"/proc/{pid}/status") as f:
+            for line in f:
+                if line.startswith(field):
+                    return round(int(line.split()[1]) / 1024, 1)
+    except OSError:
+        return None
+    return None
+
+
+def _descendants(root: int) -> list[int]:
+    """Every process under ``root``. vLLM's default CPU path runs the model in
+    a worker process, so a self-only reading misses the weights entirely."""
+    children: dict[int, list[int]] = {}
+    for entry in os.listdir("/proc"):
+        if not entry.isdigit():
+            continue
+        try:
+            with open(f"/proc/{entry}/stat") as f:
+                fields = f.read().rsplit(")", 1)[1].split()
+            ppid = int(fields[1])
+        except (OSError, IndexError, ValueError):
+            continue
+        children.setdefault(ppid, []).append(int(entry))
+    out, stack = [], [root]
+    while stack:
+        pid = stack.pop()
+        for child in children.get(pid, []):
+            out.append(child)
+            stack.append(child)
+    return out
+
+
+def _tree_anon_mb(pids: list[int]) -> float:
+    return round(sum(_rss("RssAnon:", p) or 0.0 for p in pids), 1)
+
+
+def _process_start_epoch() -> float | None:
+    """When this process was created, so imports are inside the measurement."""
+    try:
+        with open("/proc/self/stat") as f:
+            fields = f.read().rsplit(")", 1)[1].split()
+        starttime_ticks = int(fields[19])          # /proc/self/stat field 22
+        hz = os.sysconf("SC_CLK_TCK")
+        with open("/proc/stat") as f:
+            for line in f:
+                if line.startswith("btime"):
+                    return int(line.split()[1]) + starttime_ticks / hz
+    except Exception:
+        return None
+    return None
+
+
+class AnonSampler(threading.Thread):
+    """Peak RssAnon has no kernel counter, so poll for it -- across the whole
+    process tree, because the model may not be in this process."""
+
+    def __init__(self, interval: float = 0.02, rescan_every: float = 0.5):
+        super().__init__(daemon=True)
+        self.interval, self.rescan_every = interval, rescan_every
+        self.peak = self.peak_self = 0.0
+        self.marks: list[dict] = []
+        self.pids = [os.getpid()]
+        self.max_procs = 1
+        # not `_stop`: threading.Thread already defines a private _stop().
+        self._done = threading.Event()
+
+    def _refresh(self) -> None:
+        me = os.getpid()
+        self.pids = [me] + _descendants(me)
+        self.max_procs = max(self.max_procs, len(self.pids))
+
+    def _sample(self) -> tuple[float, float]:
+        mine = _rss("RssAnon:") or 0.0
+        tree = _tree_anon_mb(self.pids)
+        self.peak_self = max(self.peak_self, mine)
+        self.peak = max(self.peak, tree)
+        return mine, tree
+
+    def run(self) -> None:
+        last_scan = 0.0
+        while not self._done.is_set():
+            now = time.monotonic()
+            if now - last_scan >= self.rescan_every:
+                self._refresh()
+                last_scan = now
+            self._sample()
+            self._done.wait(self.interval)
+
+    def mark(self, name: str) -> float | None:
+        """Record a phase boundary with the peak reached so far."""
+        self._refresh()
+        mine, tree = self._sample()
+        self.marks.append({"phase": name, "anon_mb": mine,
+                           "tree_anon_mb": tree, "n_procs": len(self.pids),
+                           "peak_tree_so_far_mb": round(self.peak, 1)})
+        return mine
+
+    def stop(self) -> None:
+        self._done.set()
+        self.join(timeout=2.0)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--model", required=True)
+    ap.add_argument("--tag", required=True)
+    ap.add_argument("--max-model-len", type=int, default=2048)
+    ap.add_argument("--quantization", default=None)
+    ap.add_argument("--kv-cache-memory-bytes", type=int, default=None,
+                    help="exact KV budget; VLLM_CPU_KVCACHE_SPACE is integer GiB "
+                         "so it cannot express an edge-sized cache")
+    ap.add_argument("--max-num-batched-tokens", type=int, default=None,
+                    help="vLLM sizes its warm-up forward from this; an edge "
+                         "deployment serving one short stream does not need "
+                         "the server default")
+    ap.add_argument("--trim", action="store_true",
+                    help="return freed allocator arenas to the OS after init")
+    ap.add_argument("--generate", type=int, default=8,
+                    help="tokens to generate, to prove the engine really works")
+    ap.add_argument("--context-len", type=int, default=0,
+                    help="after init, run a request with this many prompt "
+                         "tokens so the KV cache is populated, and report "
+                         "resident memory there. 0 skips it -- and a memory "
+                         "number taken without it says nothing about cache "
+                         "sizing.")
+    ap.add_argument("--sample-interval", type=float, default=0.02)
+    ap.add_argument("--out", required=True)
+    a = ap.parse_args()
+
+    proc_start = _process_start_epoch()
+    sampler = AnonSampler(a.sample_interval)
+    sampler.start()
+
+    anon_start = sampler.mark("before_import")
+    import vllm_omni  # noqa: F401
+    from vllm import LLM, SamplingParams
+
+    anon_after_import = sampler.mark("after_import")
+    import_s = time.time() - _T_IMPORT
+
+    t0 = time.perf_counter()
+    extra = {}
+    if a.kv_cache_memory_bytes:
+        extra["kv_cache_memory_bytes"] = a.kv_cache_memory_bytes
+    if a.max_num_batched_tokens:
+        extra["max_num_batched_tokens"] = a.max_num_batched_tokens
+    llm = LLM(model=a.model, dtype="bfloat16", max_model_len=a.max_model_len,
+              max_num_seqs=1, quantization=a.quantization,
+              enable_prefix_caching=False, trust_remote_code=False,
+              disable_log_stats=True, **extra)
+    init_s = time.perf_counter() - t0
+    ready_s = (time.time() - proc_start) if proc_start else None
+
+    anon_after_engine = sampler.mark("after_engine")
+    peak_during_load = round(sampler.peak, 1)
+
+    trimmed = None
+    if a.trim:
+        # The GPTQ repack unpacks 4-bit weights to int32 (4 bytes per weight)
+        # and throws the intermediates away, but a caching allocator keeps the
+        # arenas, so they stay resident. Ask for them back.
+        import ctypes
+
+        released = []
+        for lib, sym in ((None, "malloc_trim"),
+                         ("libtcmalloc_minimal.so.4",
+                          "MallocExtension_ReleaseFreeMemory")):
+            try:
+                handle = ctypes.CDLL(lib) if lib else ctypes.CDLL("libc.so.6")
+                fn = getattr(handle, sym)
+                fn(0) if sym == "malloc_trim" else fn()
+                released.append(sym)
+            except Exception:
+                pass
+        trimmed = released
+    anon_after_trim = sampler.mark("after_trim")
+
+    out = llm.generate([{"prompt_token_ids": [10]}],
+                       SamplingParams(temperature=0.0, max_tokens=a.generate,
+                                      ignore_eos=True), use_tqdm=False)
+    n_out = len(out[0].outputs[0].token_ids)
+    sampler.mark("after_short_generate")
+
+    anon_at_context = context_tokens = None
+    if a.context_len:
+        # Fill the cache. Without this the cache is allocated and never
+        # written, so resident memory says nothing about how it was sized.
+        context_tokens = min(a.context_len, a.max_model_len - 16)
+        llm.generate([{"prompt_token_ids": [10] * context_tokens}],
+                     SamplingParams(temperature=0.0, max_tokens=16,
+                                    ignore_eos=True), use_tqdm=False)
+        anon_at_context = sampler.mark("after_context_generate")
+
+    sampler.stop()
+
+    rec = {
+        "tag": a.tag, "model": a.model,
+        "ready_s": round(ready_s, 2) if ready_s else None,
+        "import_s": round(import_s, 2),
+        "init_s": round(init_s, 2),
+        # VmRSS counts the mmap'd checkpoint's file pages, which come and go
+        # with system page-cache pressure: identical configurations measured
+        # 1.2-2.0 GB apart. RssAnon is the allocated memory -- weights copied
+        # into tensors, the KV cache, activations -- and is what a memory
+        # budget is actually about.
+        # self-only, for comparability with rows taken before the tree fix
+        "rss_anon_mb": _rss("RssAnon:"), "rss_file_mb": _rss("RssFile:"),
+        # what the device actually has to hold
+        "rss_anon_tree_mb": _tree_anon_mb([os.getpid()] + _descendants(os.getpid())),
+        "peak_rss_anon_mb": round(sampler.peak, 1),
+        "peak_rss_anon_self_mb": round(sampler.peak_self, 1),
+        "peak_rss_anon_during_load_mb": peak_during_load,
+        "n_procs_max": sampler.max_procs,
+        "anon_before_import_mb": anon_start,
+        "anon_after_import_mb": anon_after_import,
+        "anon_after_engine_mb": anon_after_engine,
+        "anon_after_trim_mb": anon_after_trim,
+        "rss_anon_at_context_mb": anon_at_context,
+        "context_tokens": context_tokens,
+        "phase_marks": sampler.marks,
+        "trim_calls": trimmed,
+        "rss_mb": _rss("VmRSS:"), "rss_peak_mb": _rss("VmHWM:"),
+        "maxrss_mb": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1),
+        "kvcache_space_gib": os.environ.get("VLLM_CPU_KVCACHE_SPACE", ""),
+        "kv_cache_memory_bytes": a.kv_cache_memory_bytes,
+        "max_num_batched_tokens": a.max_num_batched_tokens,
+        "cache_root": os.environ.get("VLLM_CACHE_ROOT", "(default)"),
+        "aot_shim": os.environ.get("VLLM_OMNI_AOT_CACHE_SHIM", "1"),
+        "max_model_len": a.max_model_len,
+        "tokens_generated": n_out,
+        "provenance": provenance({
+            "quantization": a.quantization,
+            "sample_interval_s": a.sample_interval,
+            # uni vs mp decides whether the model is in this process at all
+            "v1_multiprocessing": os.environ.get(
+                "VLLM_ENABLE_V1_MULTIPROCESSING", "1"),
+            "memory_scope": "process tree",
+        }),
+    }
+    print(json.dumps(rec, indent=2))
+    with open(a.out, "a") as f:
+        f.write(json.dumps(rec) + "\n")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/check_text_reliability.py

@@ -0,0 +1,175 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Isolated real-model stream saturation, cancellation and owned-worker failure checks."""
+
+import argparse
+import asyncio
+import copy
+import os
+import sys
+import time
+import traceback
+from pathlib import Path
+
+from profile_local_text import save
+
+
+async def run(args):
+    import psutil
+
+    from vllm_omni.edge.local.engine import LocalTextEngine
+    from vllm_omni.edge.local.plan import plan_text_session
+
+    args.out.mkdir(parents=True, exist_ok=False)
+    report = {
+        "status": "running",
+        "start_unix": time.time(),
+        "argv": sys.argv,
+        "scope": (
+            "LocalTextEngine consumer queue and actual owned engine-process failure; "
+            "not proof of every internal queue being bounded"
+        ),
+    }
+    engine = None
+    owned = []
+    save(args.out / "report.json", report)
+    try:
+        plan = plan_text_session(
+            args.model, max_model_len=4096, max_num_seqs=4, max_num_batched_tokens=512, enforce_eager=True
+        )
+        report["plan"] = plan.to_dict()
+        if not plan.admitted:
+            report["status"] = "rejected"
+            return
+        oversized = plan_text_session(
+            args.model, max_model_len=4096, max_num_seqs=100000, max_num_batched_tokens=512, enforce_eager=True
+        )
+        report["oversized_plan"] = oversized.to_dict()
+        report["oversized_plan_refused"] = not oversized.admitted
+        engine = LocalTextEngine(plan)
+        await engine.start()
+        prompt = "Describe a quiet garden with trees, flowers and a pond."
+
+        async def request(saturate=False):
+            session = engine.open_session()
+            rid, stream = await engine.submit(
+                session, prompt, max_tokens=128, temperature=0, ignore_eos=True, max_chunks=4
+            )
+            if saturate:
+                # Wait for actual capacity, rather than guessing a delay from device speed.
+                deadline = time.monotonic() + 30
+                while stream.stats()["high_water_chunks"] < 4 and time.monotonic() < deadline:
+                    await asyncio.sleep(0.05)
+                report["saturated_queue_before_drain"] = stream.stats()
+            events = [e.to_dict() async for e in stream]
+            record = engine.records[rid].to_dict()
+            record["stream"] = stream.stats()
+            record["events"] = events
+            engine.close_session(session.session_id)
+            assert record["finished"] and not record["error"] and record["output_tokens"] == 128
+            return record
+
+        report["baseline"] = await request()
+        report["saturated"] = await request(saturate=True)
+        report["saturation_reached"] = report["saturated_queue_before_drain"]["high_water_chunks"] == 4
+        report["same_tokens_after_saturation"] = (
+            report["baseline"]["output_token_ids"] == report["saturated"]["output_token_ids"]
+        )
+        session = engine.open_session()
+        stale = copy.deepcopy(session)
+        rid, stream = await engine.submit(session, prompt, max_tokens=512, ignore_eos=True, max_chunks=4)
+        deadline = time.monotonic() + 30
+        while stream.stats()["high_water_chunks"] < 4 and time.monotonic() < deadline:
+            await asyncio.sleep(0.05)
+        report["cancel_queue_before"] = stream.stats()
+        report["cancel"] = await engine.cancel(rid)
+        report["cancel"]["late_events"] = [e.to_dict() async for e in stream]
+        try:
+            await engine.submit(stale, prompt, max_tokens=1)
+        except RuntimeError as error:
+            report["stale_handle_rejected"] = str(error)
+        else:
+            report["stale_handle_rejected"] = False
+        engine.close_session(session.session_id)
+        report["after_cancel"] = await request()
+        save(args.out / "report.json", report)
+
+        # Process handles come from this engine's actual manager, then must also
+        # belong to this probe's process tree. Never select arbitrary system PIDs.
+        client = engine._omni.engine.stage_clients[0]
+        manager = client.resources.engine_manager
+        processes = [p for p in manager.processes if p.is_alive()]
+        descendants = {p.pid: p for p in psutil.Process().children(recursive=True)}
+        assert processes and all(p.pid in descendants for p in processes)
+        manager_tree = {}
+        for handle in processes:
+            core = descendants[handle.pid]
+            manager_tree[core.pid] = core
+            manager_tree.update({child.pid: child for child in core.children(recursive=True)})
+        owned = [(p.pid, p.create_time()) for p in manager_tree.values()]
+        target = processes[0]
+        session = engine.open_session()
+        rid, stream = await engine.submit(session, prompt, max_tokens=512, ignore_eos=True)
+        first = await asyncio.wait_for(stream.get(), 30)
+        assert first is not None and first.kind == "token", "No token before worker-failure injection"
+        report["crash_first"] = first.to_dict()
+        report["crash_injection"] = {
+            "pid": target.pid,
+            "name": target.name,
+            "created": descendants[target.pid].create_time(),
+            "ownership": "engine manager handle and current process descendant",
+        }
+        start = time.perf_counter()
+        target.terminate()
+
+        async def drain():
+            return [e.to_dict() async for e in stream]
+
+        try:
+            events = await asyncio.wait_for(drain(), 30)
+            report["crash_injection"].update(
+                events=events,
+                observed_within_30s=True,
+                explicit_error_reported=bool(engine.records[rid].error) or any(e["kind"] == "error" for e in events),
+                request_record=engine.records[rid].to_dict(),
+            )
+        except asyncio.TimeoutError:
+            report["crash_injection"].update(observed_within_30s=False, error="No terminal stream event within 30s")
+        report["crash_injection"]["observation_s"] = time.perf_counter() - start
+        report["status"] = "completed"
+    except Exception as error:
+        report.update(status="failed", error=repr(error), traceback=traceback.format_exc())
+        traceback.print_exc()
+    finally:
+        if engine is not None:
+            try:
+                await engine.close()
+            except (Exception, asyncio.CancelledError) as error:
+                report.update(status="failed", cleanup_error=repr(error))
+                report.setdefault("error", f"Engine shutdown failed after probe: {error!r}")
+        await asyncio.sleep(2)
+        leftovers = []
+        for pid, created in owned:
+            try:
+                p = psutil.Process(pid)
+                if p.create_time() == created and p.is_running():
+                    leftovers.append({"pid": pid, "created": created, "status": p.status()})
+                    # Contain an isolated diagnostic's leak; report it, do not call cleanup a pass.
+                    p.kill()
+            except psutil.NoSuchProcess:
+                pass
+        report["owned_processes_requiring_harness_cleanup"] = leftovers
+        report["end_unix"] = time.time()
+        save(args.out / "report.json", report)
+    return report["status"] == "completed"
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--model", required=True)
+    p.add_argument("--out", type=Path, required=True)
+    args = p.parse_args()
+    os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+    from vllm_omni.windows.aio import install_selector_policy
+
+    install_selector_policy()
+    raise SystemExit(0 if asyncio.run(run(args)) else 1)

benchmarks/edge_harness/check_tts_reliability.py

@@ -0,0 +1,172 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Real-weight TTS slow consumer, abort/recovery and owned vocoder-process failure."""
+
+import argparse
+import asyncio
+import os
+import sys
+import time
+import traceback
+from pathlib import Path
+
+from profile_local_text import save
+
+
+async def run(args):
+    import psutil
+    import torch
+    from transformers import AutoTokenizer
+
+    from vllm_omni import AsyncOmni
+    from vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts import Qwen3TTSConfig
+    from vllm_omni.model_executor.models.qwen3_tts.prompt_embeds_builder import Qwen3TTSPromptEmbedsBuilder
+
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tts"))
+    from stream_latency_bench import _cat_audio
+
+    args.out.mkdir(parents=True, exist_ok=False)
+    report = {
+        "status": "running",
+        "argv": sys.argv,
+        "start_unix": time.time(),
+        "scope": (
+            "Short real audio delivery, slow consumer, abort fence/recovery, owned vocoder termination. "
+            "Not audio quality or 64 MiB buffer saturation."
+        ),
+    }
+    omni, owned = None, []
+    save(args.out / "report.json", report)
+    try:
+        cfg = Qwen3TTSConfig.from_pretrained(args.model)
+        tok = AutoTokenizer.from_pretrained(args.model)
+
+        def prompt(text):
+            info = {"task_type": ["CustomVoice"], "text": [text], "language": ["English"], "speaker": ["Vivian"]}
+            length = Qwen3TTSPromptEmbedsBuilder.estimate_prompt_len_from_additional_information(
+                additional_information=info,
+                task_type="CustomVoice",
+                tokenize_prompt=lambda x: tok(x, padding=False)["input_ids"],
+                codec_language_id=cfg.talker_config.codec_language_id,
+                spk_is_dialect=cfg.talker_config.spk_is_dialect,
+            )
+            return {"prompt_token_ids": [0] * length, "additional_information": info}
+
+        short = prompt("The garden is quiet. We can hear the birds singing.")
+        long = prompt("This longer sentence keeps the speech model generating while we test interruption. " * 12)
+        omni = AsyncOmni(model=args.model, deploy_profile="edge", stage_init_timeout=180)
+        params = omni.resolve_sampling_params_list(None, allow_delta_coercion=True)
+
+        def event(output, rid):
+            assert output.request_id == rid, "Unexpected request ID in audio stream"
+            mm = output.outputs[0].multimodal_output if output.outputs else None
+            audio = _cat_audio(mm.get("audio") if mm else None)
+            return {
+                "request_id": output.request_id,
+                "finished": output.finished,
+                "samples": int(audio.numel()) if audio is not None else 0,
+                "finite": bool(torch.isfinite(audio).all()) if audio is not None else True,
+            }
+
+        async def first_audio(stream, rid):
+            async for output in stream:
+                row = event(output, rid)
+                if row["samples"]:
+                    assert row["finite"], "Nonfinite audio before fault probe"
+                    return row
+            raise RuntimeError("No audio before fault probe")
+
+        async def collect(stream, rid, pause=False):
+            rows = []
+            async for output in stream:
+                rows.append(event(output, rid))
+                if pause and len(rows) == 1:
+                    await asyncio.sleep(3)
+            return rows
+
+        for rid, slow in [("baseline", False), ("slow-consumer", True)]:
+            rows = await collect(omni.generate(short, request_id=rid, sampling_params_list=params), rid, slow)
+            report[rid] = rows
+            assert any(r["finished"] for r in rows) and any(r["samples"] for r in rows)
+            assert all(r["finite"] for r in rows)
+        rid = "abort-probe"
+        stream = omni.generate(long, request_id=rid, sampling_params_list=params)
+        first = await asyncio.wait_for(first_audio(stream, rid), 60)
+        start = time.perf_counter()
+        await omni.abort(rid)
+        report["abort"] = {"first": first, "ack_s": time.perf_counter() - start}
+        try:
+            late = await asyncio.wait_for(collect(stream, rid), 10)
+            report["abort"]["late_events"] = late
+            report["abort"]["late_audio_samples"] = sum(r["samples"] for r in late)
+        except asyncio.CancelledError:
+            report["abort"]["terminal"] = "CancelledError"
+        except Exception as error:
+            report["abort"]["terminal_error"] = repr(error)
+        rid = "after-abort"
+        rows = await collect(omni.generate(short, request_id=rid, sampling_params_list=params), rid)
+        report[rid] = rows
+        assert any(r["finished"] for r in rows) and any(r["samples"] for r in rows)
+        save(args.out / "report.json", report)
+        descendants = {p.pid: p for p in psutil.Process().children(recursive=True)}
+        core_handles = [
+            p for client in omni.engine.stage_clients for p in client.resources.engine_manager.processes if p.is_alive()
+        ]
+        assert core_handles and all(p.pid in descendants for p in core_handles)
+        manager_tree = {}
+        for handle in core_handles:
+            core = descendants[handle.pid]
+            manager_tree[core.pid] = core
+            manager_tree.update({p.pid: p for p in core.children(recursive=True)})
+        owned = [(p.pid, p.create_time()) for p in manager_tree.values()]
+        target = omni.engine.stage_clients[-1].resources.engine_manager.processes[0]
+        rid = "vocoder-crash"
+        stream = omni.generate(long, request_id=rid, sampling_params_list=params)
+        report["crash_first"] = await asyncio.wait_for(first_audio(stream, rid), 60)
+        report["crash"] = {"pid": target.pid, "name": target.name, "created": descendants[target.pid].create_time()}
+        start = time.perf_counter()
+        target.terminate()
+        try:
+            report["crash"]["events"] = await asyncio.wait_for(collect(stream, rid), 30)
+            report["crash"]["explicit_error_reported"] = False
+        except asyncio.TimeoutError:
+            report["crash"]["timeout"] = True
+        except (Exception, asyncio.CancelledError) as error:
+            report["crash"].update(explicit_error_reported=True, error=repr(error))
+        report["crash"]["observation_s"] = time.perf_counter() - start
+        report["status"] = "completed"
+    except Exception as error:
+        report.update(status="failed", error=repr(error), traceback=traceback.format_exc())
+        traceback.print_exc()
+    finally:
+        if omni is not None:
+            try:
+                omni.shutdown()
+            except (Exception, asyncio.CancelledError) as error:
+                report.update(status="failed", cleanup_error=repr(error))
+                report.setdefault("error", f"Engine shutdown failed after probe: {error!r}")
+        await asyncio.sleep(2)
+        leftovers = []
+        for pid, created in owned:
+            try:
+                p = psutil.Process(pid)
+                if p.create_time() == created and p.is_running():
+                    leftovers.append({"pid": pid, "created": created, "status": p.status()})
+                    p.kill()
+            except psutil.NoSuchProcess:
+                pass
+        report["owned_processes_requiring_harness_cleanup"] = leftovers
+        report["end_unix"] = time.time()
+        save(args.out / "report.json", report)
+    return report["status"] == "completed"
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--model", required=True)
+    p.add_argument("--out", type=Path, required=True)
+    args = p.parse_args()
+    os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+    from vllm_omni.windows.aio import install_selector_policy
+
+    install_selector_policy()
+    raise SystemExit(0 if asyncio.run(run(args)) else 1)

benchmarks/edge_harness/download_pending_weights.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Resume the missing E2E checkpoints and verify every safetensors SHA-256.
+
+Weights and status files stay outside the Git checkout. This helper can wait for
+an already-running Hugging Face transfer before resuming it, so an interrupted
+interactive session does not leave an unverified partial checkpoint behind.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from huggingface_hub import HfApi, snapshot_download
+
+
+@dataclass(frozen=True)
+class Checkpoint:
+    repo: str
+    revision: str
+    directory: str
+
+
+CHECKPOINTS = {
+    "minicpmo": Checkpoint(
+        "openbmb/MiniCPM-o-4_5",
+        "503e754207c94da6bb26850b4469f367c9ea3582",
+        "MiniCPM-o-4_5",
+    ),
+    "internvla_base": Checkpoint(
+        "InternRobotics/InternVLA-A1-3B",
+        "08cf716d413aa8208a34f346cd8824e6ff901205",
+        "InternVLA-A1-3B",
+    ),
+    "internvla_place_markpen": Checkpoint(
+        "Jia-Zeng/InternVLA-A1-3B-FineTuned-Place_Markpen",
+        "0a557c6a503e6545b369bff2f34c76a410bbd190",
+        "InternVLA-A1-3B-FT-Place_Markpen",
+    ),
+}
+
+
+def _status(path: Path, **fields: object) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(".json.tmp")
+    temporary.write_text(json.dumps(fields, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    temporary.replace(path)
+
+
+def _wait_for_existing(pid: int | None, repo: str) -> None:
+    if pid is None:
+        return
+    command_path = Path(f"/proc/{pid}/cmdline")
+    while command_path.exists():
+        try:
+            command = command_path.read_bytes()
+        except (FileNotFoundError, PermissionError):
+            break
+        if repo.encode() not in command:
+            break
+        time.sleep(30)
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _verify(checkpoint: Checkpoint, directory: Path) -> list[dict[str, object]]:
+    info = HfApi().model_info(checkpoint.repo, revision=checkpoint.revision, files_metadata=True)
+    weights = [sibling for sibling in info.siblings if sibling.rfilename.endswith(".safetensors")]
+    if not weights:
+        raise RuntimeError(f"No safetensors weight files advertised by {checkpoint.repo}")
+    verified = []
+    for sibling in weights:
+        path = directory / sibling.rfilename
+        expected_size = sibling.size
+        expected_hash = sibling.lfs.sha256 if sibling.lfs else None
+        if expected_size is None or expected_hash is None:
+            raise RuntimeError(f"Missing size or SHA-256 metadata for {sibling.rfilename}")
+        if not path.is_file() or path.stat().st_size != expected_size:
+            raise RuntimeError(f"Missing or incomplete weight: {path}")
+        actual_hash = _sha256(path)
+        if actual_hash != expected_hash:
+            raise RuntimeError(f"SHA-256 mismatch for {path}: {actual_hash} != {expected_hash}")
+        verified.append({"file": sibling.rfilename, "bytes": expected_size, "sha256": actual_hash})
+        print(f"Verified {path.name}: {expected_size} bytes", flush=True)
+    return verified
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("model", choices=CHECKPOINTS)
+    parser.add_argument("--model-root", type=Path, required=True)
+    parser.add_argument("--wait-pid", type=int)
+    parser.add_argument("--detach", action="store_true")
+    args = parser.parse_args()
+
+    checkpoint = CHECKPOINTS[args.model]
+    root = args.model_root.resolve()
+    status_path = root / ".weight_download_status" / f"{args.model}.json"
+    if args.detach:
+        log_path = status_path.with_suffix(".log")
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        command = [sys.executable, str(Path(__file__).resolve()), args.model, "--model-root", str(root)]
+        if args.wait_pid is not None:
+            command.extend(["--wait-pid", str(args.wait_pid)])
+        with log_path.open("ab") as log:
+            process = subprocess.Popen(  # noqa: S603
+                command,
+                stdin=subprocess.DEVNULL,
+                stdout=log,
+                stderr=subprocess.STDOUT,
+                start_new_session=True,
+                close_fds=True,
+                env={**os.environ, "HF_XET_HIGH_PERFORMANCE": "1"},
+            )
+        print(json.dumps({"pid": process.pid, "log": str(log_path)}))
+        return
+
+    directory = root / checkpoint.directory
+    _status(status_path, status="waiting", repo=checkpoint.repo, revision=checkpoint.revision)
+    try:
+        _wait_for_existing(args.wait_pid, checkpoint.repo)
+        _status(status_path, status="downloading", repo=checkpoint.repo, revision=checkpoint.revision)
+        snapshot_download(
+            repo_id=checkpoint.repo,
+            revision=checkpoint.revision,
+            local_dir=str(directory),
+            max_workers=4,
+        )
+        _status(status_path, status="verifying", repo=checkpoint.repo, revision=checkpoint.revision)
+        files = _verify(checkpoint, directory)
+        _status(status_path, status="verified", repo=checkpoint.repo, revision=checkpoint.revision, files=files)
+    except Exception as exc:
+        _status(status_path, status="failed", repo=checkpoint.repo, revision=checkpoint.revision, error=str(exc))
+        raise
+
+
+if __name__ == "__main__":
+    main()

benchmarks/edge_harness/experiments/README.md

@@ -0,0 +1,20 @@
+# One-off experiment drivers
+
+Scripts that produced a specific result rather than tooling meant to be reused.
+Kept because the results in `../results/` are only checkable if the script that
+made them is here.
+
+| group | what |
+|---|---|
+| `aihub_*.py` | Qualcomm AI Hub on-device runs — Snapdragon layer timings, calibration, w8a16/w4a16 parity. Authentication comes from `~/.qai_hub/client.ini`; no credential is in these files |
+| `bench_int4_kernels.py` | the 4-bit GEMM microbenchmark — kernel-level shapes, used to size decisions before building anything |
+| `quality_int4.py`, `ppl_vllm.py`, `parity_hf_vs_vllm.py`, `ref_hf.py` | fidelity: greedy divergence, perplexity, and parity against a HuggingFace reference |
+| `quantize_int8.py`, `export_gptq_cpu.py`, `dequantize_int4.py`, `verify_export.py` | building and round-tripping checkpoints |
+| `capture_real_acts.py`, `capture_lm_head.py` | activation capture for calibrated quantization |
+| `cpu_busy.py` | a load generator, used to test that the harness's contention guard actually refuses |
+
+**They are not portable as written.** Most carry absolute paths from the machine
+they ran on (`/data/<user>/embedding_infer/...`) and expect checkpoints under
+`models/`. They are a record of what was run, not a supported interface — the
+supported entry points are the tools one level up, described in
+[docs/edge/harness.md](../../../docs/edge/harness.md).

benchmarks/edge_harness/experiments/aihub_block4.py

@@ -0,0 +1,50 @@
+"""The architecture's repeating 4-layer block (3 sliding + 1 full) on a real S25.
+
+Profiling one layer at a time pays the graph's Input/Output boundary once per
+layer. A real decode step pays it once for all 28. Measuring 1 layer and 4
+layers at the same context separates the fixed per-graph cost from the true
+per-layer cost, so the 28-layer step can be extrapolated honestly.
+"""
+import json
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+OPT = {
+    "fp16": "--target_runtime qnn_context_binary --quantize_full_type float16",
+    "w4a16": "--target_runtime qnn_context_binary --quantize_full_type w4a16 --quantize_io",
+}
+
+res, pend = {}, {}
+mdl = hub.upload_model(f"{HERE}/onnx/ring_block4_ctx1024.onnx", name="spark25_block4")
+print("uploaded", flush=True)
+for q in ("w4a16", "fp16"):
+    try:
+        cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                                    name=f"spark block4 {q}", options=OPT[q])
+        pend[q] = cj
+        print("compile", q, cj.job_id, flush=True)
+    except Exception as e:
+        print("submit failed", q, str(e)[:200], flush=True)
+
+prof = {}
+for q, cj in pend.items():
+    st = cj.wait()
+    print("compile", q, st.code, (st.message or "")[:200], flush=True)
+    if st.success:
+        prof[q] = hub.submit_profile_job(model=cj.get_target_model(),
+                                         device=hub.Device(DEV),
+                                         name=f"spark block4 {q}",
+                                         options="--compute_unit npu")
+for q, pj in prof.items():
+    st = pj.wait()
+    if st.success:
+        p = pj.download_profile()
+        res[q] = p["execution_summary"]["estimated_inference_time"] / 1e3
+        json.dump(p, open(f"{HERE}/profile_block4_{q}.json", "w"))
+        print(f"  block4 {q}: {res[q]:.3f} ms", flush=True)
+    else:
+        res[q] = None
+        print(f"  block4 {q} FAILED: {(st.message or '')[:250]}", flush=True)
+    json.dump(res, open(f"{HERE}/aihub_block4.json", "w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_calib_any.py

@@ -0,0 +1,78 @@
+"""Calibrated quantisation + on-device parity for any exported graph."""
+import argparse, json
+import numpy as np
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+SCHEMES = {
+    "w8a16": (hub.QuantizeDtype.INT8, hub.QuantizeDtype.INT16),
+    "w16a16": (hub.QuantizeDtype.INT16, hub.QuantizeDtype.INT16),
+}
+
+
+def snr(a, b):
+    a, b = np.asarray(a).ravel(), np.asarray(b).ravel()
+    return float(20 * np.log10(np.linalg.norm(a) / max(np.linalg.norm(a - b), 1e-12)))
+
+
+ap = argparse.ArgumentParser()
+ap.add_argument("--onnx", required=True)
+ap.add_argument("--acts", required=True)
+ap.add_argument("--map", default="", help="onnx_name=npz_name,...")
+ap.add_argument("--tag", required=True)
+ap.add_argument("--out", required=True)
+a = ap.parse_args()
+
+import onnx
+g = onnx.load(a.onnx, load_external_data=False)
+live = [t.name for t in g.graph.input]
+mapping = dict(p.split("=") for p in a.map.split(",") if p)
+d = np.load(a.acts)
+n = d["x"].shape[0]
+calib = {name: [d[mapping.get(name, name)][i] for i in range(n)] for name in live}
+probe = {name: [d[mapping.get(name, name)][0]] for name in live}
+ref_out = d["hidden_out"][0]
+
+src = hub.upload_model(a.onnx, name=f"spark25_{a.tag}_src")
+qj = {}
+for tag, (w, act) in SCHEMES.items():
+    qj[tag] = hub.submit_quantize_job(model=src, calibration_data=calib,
+                                      weights_dtype=w, activations_dtype=act,
+                                      name=f"spark {a.tag} {tag}")
+    print("quantize", tag, qj[tag].job_id, flush=True)
+
+res, cj = {}, {}
+for tag, j in qj.items():
+    st = j.wait()
+    print("quantize", tag, st.code, (st.message or "")[:140], flush=True)
+    if st.success:
+        cj[tag] = hub.submit_compile_job(
+            model=j.get_target_model(), device=hub.Device(DEV),
+            name=f"spark {a.tag} {tag}",
+            options="--target_runtime qnn_context_binary --quantize_io")
+ij, pj = {}, {}
+for tag, j in cj.items():
+    st = j.wait()
+    print("compile", tag, st.code, (st.message or "")[:140], flush=True)
+    if st.success:
+        t = j.get_target_model()
+        ij[tag] = hub.submit_inference_job(model=t, device=hub.Device(DEV),
+                                           inputs=probe, name=f"spark {a.tag} {tag}")
+        pj[tag] = hub.submit_profile_job(model=t, device=hub.Device(DEV),
+                                         name=f"spark {a.tag} {tag}",
+                                         options="--compute_unit npu")
+for tag in list(ij):
+    e = res.setdefault(tag, {})
+    st = ij[tag].wait()
+    if st.success:
+        out = ij[tag].download_output_data()
+        e["snr_db"] = round(snr(ref_out, np.asarray(list(out.values())[0][0])), 2)
+    else:
+        e["snr_db"] = None
+    st = pj[tag].wait()
+    e["ms"] = (pj[tag].download_profile()["execution_summary"]
+               ["estimated_inference_time"] / 1e3) if st.success else None
+    print(f"  {a.tag} {tag}: {e}", flush=True)
+    json.dump(res, open(a.out, "w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_calibrated.py

@@ -0,0 +1,85 @@
+"""Calibrated quantisation of the sliding layer, judged on real activations.
+
+Uncalibrated w4a16/w4a8 scored -1.6 / -1.28 dB against the reference -- output
+uncorrelated with the truth. This feeds the quantiser real activations
+captured from the reference model on real text, then measures the same way.
+"""
+import json
+import numpy as np
+import qai_hub as hub
+
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+NAMES = ["x", "cos_sw", "sin_sw", "mask_sw", "k_cache_0", "v_cache_0"]
+
+d = np.load(f"{HERE}/ring3_real_acts.npz")
+n_samples = d["x"].shape[0]
+calib = {n: [d[n][i] for i in range(n_samples)] for n in NAMES}
+probe = {n: [d[n][0]] for n in NAMES}
+ref_out = d["hidden_out"][0]
+
+SCHEMES = {
+    "w4a16": (hub.QuantizeDtype.INT4, hub.QuantizeDtype.INT16),
+    "w8a16": (hub.QuantizeDtype.INT8, hub.QuantizeDtype.INT16),
+    "w4a8": (hub.QuantizeDtype.INT4, hub.QuantizeDtype.INT8),
+    "w8a8": (hub.QuantizeDtype.INT8, hub.QuantizeDtype.INT8),
+}
+
+
+def snr(a, b):
+    a, b = np.asarray(a).ravel(), np.asarray(b).ravel()
+    return float(20 * np.log10(np.linalg.norm(a) / max(np.linalg.norm(a - b), 1e-12)))
+
+
+src = hub.upload_model(f"{HERE}/onnx/ring3_sliding.onnx", name="spark25_calib_src")
+qjobs = {}
+for tag, (w, a) in SCHEMES.items():
+    qj = hub.submit_quantize_job(model=src, calibration_data=calib,
+                                 weights_dtype=w, activations_dtype=a,
+                                 name=f"spark calib {tag}")
+    qjobs[tag] = qj
+    print("quantize", tag, qj.job_id, flush=True)
+
+res = {}
+cjobs = {}
+for tag, qj in qjobs.items():
+    st = qj.wait()
+    print("quantize", tag, st.code, (st.message or "")[:150], flush=True)
+    if st.success:
+        cjobs[tag] = hub.submit_compile_job(
+            model=qj.get_target_model(), device=hub.Device(DEV),
+            name=f"spark calib {tag}",
+            options="--target_runtime qnn_context_binary --quantize_io")
+
+ijobs, pjobs = {}, {}
+for tag, cj in cjobs.items():
+    st = cj.wait()
+    print("compile", tag, st.code, (st.message or "")[:150], flush=True)
+    if st.success:
+        tgt = cj.get_target_model()
+        ijobs[tag] = hub.submit_inference_job(model=tgt, device=hub.Device(DEV),
+                                              inputs=probe, name=f"spark calib {tag}")
+        pjobs[tag] = hub.submit_profile_job(model=tgt, device=hub.Device(DEV),
+                                            name=f"spark calib {tag}",
+                                            options="--compute_unit npu")
+
+for tag in list(ijobs):
+    st = ijobs[tag].wait()
+    entry = res.setdefault(tag, {})
+    if st.success:
+        out = ijobs[tag].download_output_data()
+        got = np.asarray(list(out.values())[0][0])
+        entry["snr_db"] = round(snr(ref_out, got), 2)
+    else:
+        entry["snr_db"] = None
+        print(tag, "inference FAILED", (st.message or "")[:150], flush=True)
+    st = pjobs[tag].wait()
+    if st.success:
+        p = pjobs[tag].download_profile()
+        entry["ms"] = p["execution_summary"]["estimated_inference_time"] / 1e3
+        json.dump(p, open(f"{HERE}/profile_calib_{tag}.json", "w"))
+    else:
+        entry["ms"] = None
+    print(f"  {tag}: {entry}", flush=True)
+    json.dump(res, open(f"{HERE}/aihub_calibrated.json", "w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_cpu_int8.py

@@ -0,0 +1,40 @@
+"""A quantized phone-CPU reference for the same layer.
+
+llama.cpp cannot be run on an AI Hub device, and it would use 4-bit weights on
+the CPU, so comparing our NPU path against an fp16 CPU run would flatter us.
+This asks for the closest thing AI Hub offers: the same graph, int8, on the
+S25's CPU.
+"""
+import json
+import qai_hub as hub
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+CAND = [
+    ("tflite_int8_cpu", "--target_runtime tflite --quantize_full_type int8 --quantize_io", "cpu"),
+    ("tflite_w8a8_cpu", "--target_runtime tflite --quantize_full_type w8a8 --quantize_io", "cpu"),
+]
+res, pend = {}, {}
+mdl = hub.upload_model(f"{HERE}/onnx/ring3_sliding.onnx", name="spark25_ring3_cpu")
+for name, opt, unit in CAND:
+    try:
+        cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                                    name=f"spark {name}", options=opt)
+        pend[(name, unit)] = cj; print("compile", name, cj.job_id, flush=True)
+    except Exception as e:
+        print("submit failed", name, str(e)[:180], flush=True)
+prof = {}
+for key, cj in pend.items():
+    st = cj.wait(); print("compile", key[0], st.code, (st.message or "")[:180], flush=True)
+    if st.success:
+        prof[key] = hub.submit_profile_job(model=cj.get_target_model(), device=hub.Device(DEV),
+                                           name=f"spark {key[0]}", options=f"--compute_unit {key[1]}")
+for key, pj in prof.items():
+    st = pj.wait()
+    if st.success:
+        p = pj.download_profile()
+        res[key[0]] = p["execution_summary"]["estimated_inference_time"]/1e3
+        print(f"  {key[0]}: {res[key[0]]:.3f} ms", flush=True)
+    else:
+        res[key[0]] = None; print(f"  {key[0]} FAILED:", (st.message or "")[:180], flush=True)
+    json.dump(res, open(f"{HERE}/aihub_cpu_int8.json","w"), indent=1)
+print("done", flush=True)

benchmarks/edge_harness/experiments/aihub_gelu.py

@@ -0,0 +1,31 @@
+"""Is the cheaper GELU actually cheaper on the Hexagon NPU?
+
+The erf form is 11-13% of every layer in the device profile. tanh holds 65.2 dB
+against the reference with top-1/top-5 intact, so it is adoptable if it buys
+time. This measures whether it does.
+"""
+import json
+import qai_hub as hub
+HERE = "/data/zhoutaichang/embedding_infer/analysis/experiments/spark_edge"
+DEV = "Samsung Galaxy S25"
+OPT = "--target_runtime qnn_context_binary --quantize_full_type float16"
+res = {}
+mdl = hub.upload_model(f"{HERE}/onnx/ring_sliding_tanh.onnx", name="spark25_sliding_tanh")
+cj = hub.submit_compile_job(model=mdl, device=hub.Device(DEV),
+                            name="spark sliding ring tanh fp16", options=OPT)
+print("compile", cj.job_id, flush=True)
+st = cj.wait(); print("compile", st.code, (st.message or "")[:160], flush=True)
+if st.success:
+    pj = hub.submit_profile_job(model=cj.get_target_model(), device=hub.Device(DEV),
+                                name="spark sliding ring tanh fp16",
+                                options="--compute_unit npu")
+    st2 = pj.wait()
+    if st2.success:
+        p = pj.download_profile()
+        res["ring_sliding_tanh_fp16"] = p["execution_summary"]["estimated_inference_time"] / 1e3
+        json.dump(p, open(f"{HERE}/profile_ring_sliding_tanh_fp16.json", "w"))
+        print(f"  tanh: {res['ring_sliding_tanh_fp16']:.3f} ms", flush=True)
+    else:
+        print("  profile FAILED:", (st2.message or "")[:200], flush=True)
+json.dump(res, open(f"{HERE}/aihub_gelu.json", "w"), indent=1)
+print("done", flush=True)