vllm-omni/tests/edge/test_qnn_export.py

"""WP4/mobile: the decode-step graph exported for phone accelerators (QNN / TFLite / ORT).

Checks the maths of the grouped-query attention layout against a straightforward reference, the KV-cache contract,
and that the exported ONNX is free of the three constructs the Qualcomm toolchains reject.
"""

from __future__ import annotations

import importlib.util
import math

import pytest
import torch

from vllm_omni.edge import qnn_export as qx

pytestmark = [pytest.mark.core_model, pytest.mark.cpu]

CFG = qx.DecodeStepConfig(
    hidden_size=32,
    num_hidden_layers=2,
    num_attention_heads=4,
    num_key_value_heads=2,
    head_dim=8,
    intermediate_size=48,
    vocab_size=16,
    rope_theta=10000.0,
)
L = 6


def _module(seed: int = 0) -> qx.DecodeStep:
    torch.manual_seed(seed)
    module = qx.DecodeStep(CFG).eval()
    for p in module.parameters():
        torch.nn.init.normal_(p, std=0.15)
    return module


def test_config_rejects_head_counts_that_do_not_group():
    with pytest.raises(ValueError):
        qx.DecodeStepConfig(hidden_size=32, num_hidden_layers=1, num_attention_heads=6, num_key_value_heads=4, head_dim=8, intermediate_size=48, vocab_size=16)


def test_names_and_example_input_shapes_line_up():
    names, outs = qx.input_names(CFG), qx.output_names(CFG)
    args = qx.example_inputs(CFG, L)
    assert len(names) == len(args) == 4 + 2 * CFG.num_hidden_layers
    assert names[:4] == ["x", "cos", "sin", "mask"] and names[4:6] == ["k_cache_0", "v_cache_0"]
    assert outs[0] == "logits" and outs[1:3] == ["k_new_0", "v_new_0"]
    x, cos, sin, mask, *caches = args
    assert x.shape == (1, 1, CFG.hidden_size) and cos.shape == sin.shape == (1, 1, 1, CFG.head_dim)
    assert mask.shape == (1, 1, 1, L + 1)
    assert all(c.shape == (1, CFG.num_key_value_heads, L, CFG.head_dim) for c in caches)


def test_output_contract():
    module = _module()
    out = module(*qx.example_inputs(CFG, L))
    assert len(out) == 1 + 2 * CFG.num_hidden_layers
    assert out[0].shape == (1, CFG.vocab_size)
    assert all(t.shape == (1, CFG.num_key_value_heads, 1, CFG.head_dim) for t in out[1:])


def test_wrong_number_of_caches_is_rejected():
    module = _module()
    x, cos, sin, mask, *caches = qx.example_inputs(CFG, L)
    with pytest.raises(ValueError):
        module(x, cos, sin, mask, *caches[:-1])


def test_grouped_query_layout_matches_expanded_reference():
    """The exported layout reshapes q; the reference expands k/v. They must agree."""
    module = _module(1)
    x, cos, sin, mask, *caches = qx.example_inputs(CFG, L)
    layer = module.layers[0]
    got, k_new, v_new = layer(x, cos, sin, caches[0], caches[1], mask)

    d, kv, g = CFG.head_dim, CFG.num_key_value_heads, CFG.group_size
    h = qx._rms(x, layer.input_layernorm, CFG.rms_norm_eps)
    q = qx._rope(qx._rms(layer.q_proj(h).view(1, CFG.num_attention_heads, 1, d), layer.q_norm, CFG.rms_norm_eps), cos, sin)
    k = qx._rope(qx._rms(layer.k_proj(h).view(1, kv, 1, d), layer.k_norm, CFG.rms_norm_eps), cos, sin)
    v = layer.v_proj(h).view(1, kv, 1, d)
    keys = torch.cat([caches[0], k], 2).repeat_interleave(g, dim=1)
    vals = torch.cat([caches[1], v], 2).repeat_interleave(g, dim=1)
    scores = torch.matmul(q, keys.transpose(2, 3)) / math.sqrt(d) + mask
    attn = torch.matmul(torch.softmax(scores, -1), vals).transpose(1, 2).reshape(1, 1, CFG.num_attention_heads * d)
    expected = x + layer.o_proj(attn)
    hh = qx._rms(expected, layer.post_attention_layernorm, CFG.rms_norm_eps)
    expected = expected + layer.down_proj(torch.nn.functional.silu(layer.gate_proj(hh)) * layer.up_proj(hh))

    assert torch.allclose(got, expected, atol=1e-5)
    assert torch.allclose(k_new, k) and torch.allclose(v_new, v)


def test_masked_cache_slots_do_not_change_the_result():
    """Slots the mask disables may hold anything: that is what lets the host keep one fixed-size cache."""
    module = _module(2)
    args = list(qx.example_inputs(CFG, L, position=2))
    first = module(*args)[0]
    for i in range(4, len(args)):
        cache = args[i].clone()
        cache[:, :, 2:L] = 7.0  # masked region
        args[i] = cache
    assert torch.allclose(first, module(*args)[0], atol=1e-5)


def test_placement_table_prefers_npu_for_the_step_and_gpu_for_the_vocoder():
    p = qx.PLACEMENT_MS_PER_FRAME
    assert p["talker_step"]["npu"] < p["talker_step"]["gpu"]
    assert p["code2wav_chunk"]["gpu"] < p["code2wav_chunk"]["npu"]


@pytest.mark.skipif(importlib.util.find_spec("onnx") is None, reason="onnx not installed")
def test_export_onnx_is_clean_for_the_qualcomm_toolchains(tmp_path):
    import onnx

    module = _module(3)
    manifest = qx.export_onnx(module, L, tmp_path / "step.onnx")
    assert manifest["cache_len"] == L and manifest["config"]["hidden_size"] == CFG.hidden_size
    model = onnx.load(str(tmp_path / "step.onnx"))
    onnx.checker.check_model(model, full_check=True)
    assert [i.name for i in model.graph.input] == qx.input_names(CFG)
    assert [o.name for o in model.graph.output] == qx.output_names(CFG)
    assert not [n for n in model.graph.node if n.op_type == "IsNaN"]
    assert not [n for n in model.graph.node for a in n.attribute if n.op_type == "Reshape" and a.name == "allowzero" and a.i]
    inits = {i.name: i for i in model.graph.initializer}
    assert not [n for n in model.graph.node if n.op_type in ("Unsqueeze", "Squeeze") and len(n.input) > 1 and n.input[1] in inits and not inits[n.input[1]].dims]


@pytest.mark.skipif(importlib.util.find_spec("onnxruntime") is None or importlib.util.find_spec("onnx") is None, reason="onnx/onnxruntime not installed")
def test_exported_graph_reproduces_eager(tmp_path):
    import onnxruntime as ort

    module = _module(4)
    args = qx.example_inputs(CFG, L)
    with torch.inference_mode():
        expected = module(*args)
    qx.export_onnx(module, L, tmp_path / "step.onnx")
    sess = ort.InferenceSession(str(tmp_path / "step.onnx"), providers=["CPUExecutionProvider"])
    feed = {name: arg.numpy() for name, arg in zip(qx.input_names(CFG), args)}
    got = sess.run(None, feed)
    for name, want, have in zip(qx.output_names(CFG), expected, got):
        assert torch.allclose(want, torch.from_numpy(have), atol=1e-4), name