[Kernel] Add typed SAGE for TRTLLM_ATTN (#5509)
Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
1 parent
d33c905d80
commit
896f05e7e6
5 files changed
+301
-94
No files matched your search
@@ -52,6 +52,44 @@ class TestAttentionSpec:
|
||||
with pytest.raises(ValueError, match="only supported by the TRTLLM_ATTN"):
|
||||
AttentionSpec(backend="TORCH_SDPA", skip_softmax={"target_sparsity": 0.5})
|
||||
|
||||
def test_quant_serialized_with_defaults_and_overrides(self):
|
||||
assert AttentionSpec(backend="TRTLLM_ATTN", quant={"dtype_qk": "fp8_e4m3"}).backend_kwargs()["quant"] == {
|
||||
"dtype_qk": "fp8_e4m3",
|
||||
"q_block_size": 1,
|
||||
"k_block_size": 16,
|
||||
}
|
||||
assert AttentionSpec(
|
||||
backend="TRTLLM_ATTN", quant={"dtype_qk": "int8", "q_block_size": 4, "k_block_size": 16}
|
||||
).backend_kwargs()["quant"] == {"dtype_qk": "int8", "q_block_size": 4, "k_block_size": 16}
|
||||
|
||||
def test_quant_superset_fields_passed_through(self):
|
||||
# FLASHINFER_ATTN-style config (dtype_vo / flashinfer_backend) round-trips on the shared spec.
|
||||
bk = AttentionSpec(
|
||||
backend="FLASHINFER_ATTN",
|
||||
quant={"dtype_qk": "bfloat16", "dtype_vo": "fp8_e4m3", "flashinfer_backend": "trtllm-gen"},
|
||||
).backend_kwargs()["quant"]
|
||||
assert bk["dtype_qk"] == "bfloat16"
|
||||
assert bk["dtype_vo"] == "fp8_e4m3"
|
||||
assert bk["flashinfer_backend"] == "trtllm-gen"
|
||||
|
||||
def test_quant_and_skip_softmax_coexist(self):
|
||||
bk = AttentionSpec(
|
||||
backend="TRTLLM_ATTN", quant={"dtype_qk": "int8"}, skip_softmax={"target_sparsity": 0.5}
|
||||
).backend_kwargs()
|
||||
assert bk["target_sparsity"] == 0.5 and bk["quant"]["dtype_qk"] == "int8"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"spec, match",
|
||||
[
|
||||
({"backend": "TORCH_SDPA", "quant": {"dtype_qk": "int8"}}, "only supported by the TRTLLM_ATTN"),
|
||||
({"backend": "TRTLLM_ATTN", "quant": {"dtype_qk": "int4"}}, "quant.dtype_qk"),
|
||||
({"backend": "TRTLLM_ATTN", "quant": {"dtype_qk": "int8", "k_block_size": 8}}, "quant.k_block_size"),
|
||||
],
|
||||
)
|
||||
def test_quant_validation_rejects(self, spec, match):
|
||||
with pytest.raises(ValueError, match=match):
|
||||
AttentionSpec(**spec)
|
||||
|
||||
|
||||
class TestAttentionConfig:
|
||||
def test_empty_config(self):
|
||||
|
||||
Reference in new issue
Block a user