[Bugfix][Higgs-Audio-V3] Disable XQA decode (#6068)
Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: Sysy <1370724210@qq.com>
This commit is contained in:
8 files changed
+86
-1
No files matched your search
@@ -471,6 +471,7 @@ def test_sub_config_fields_match_structured_scopes():
|
||||
"trust_remote_code",
|
||||
"dtype",
|
||||
"attention_backend",
|
||||
"attention_config",
|
||||
"moe_backend",
|
||||
"hf_overrides",
|
||||
"limit_mm_per_prompt",
|
||||
|
||||
@@ -139,6 +139,7 @@ def _engine_arg_inputs(tmp_path: Path) -> tuple[PipelineConfig, DeployConfig, st
|
||||
engine_extras={
|
||||
"model": str(stage_model),
|
||||
"attention_backend": "FLASHINFER",
|
||||
"attention_config": {"use_trtllm_attention": False},
|
||||
"hf_overrides": {"rope_scaling": {"factor": 2}},
|
||||
"limit_mm_per_prompt": {"audio": 1},
|
||||
"logits_processors": ["test.custom.LogitsProcessor"],
|
||||
@@ -229,6 +230,7 @@ def test_typed_llm_engine_args_preserve_legacy_adapter_behavior(tmp_path):
|
||||
assert thinker_args["omni_kv_config"] == {"need_send_cache": True}
|
||||
assert thinker_args["worker_cls"] == "test.custom.Worker"
|
||||
assert thinker_args["attention_backend"] == "FLASHINFER"
|
||||
assert thinker_args["attention_config"] == {"use_trtllm_attention": False}
|
||||
assert thinker_args["trust_remote_code"] is True
|
||||
assert thinker_args["dtype"] == "float16"
|
||||
assert thinker_args["distributed_executor_backend"] == "mp"
|
||||
|
||||
@@ -1326,7 +1326,9 @@ class TestRegistry:
|
||||
|
||||
assert "higgs_multimodal_qwen3" in OMNI_PIPELINES
|
||||
|
||||
def test_deploy_yaml_exists(self):
|
||||
def test_deploy_yaml_uses_native_flashinfer(self):
|
||||
from vllm_omni.config.stage_config import load_deploy_config
|
||||
|
||||
for name in (
|
||||
"higgs_multimodal_qwen3.yaml",
|
||||
"higgs_multimodal_qwen3_high_throughput.yaml",
|
||||
@@ -1334,6 +1336,10 @@ class TestRegistry:
|
||||
):
|
||||
yaml_path = get_deploy_config_path(name)
|
||||
assert os.path.isfile(yaml_path), f"Deploy YAML not found at {yaml_path}"
|
||||
deploy_config = load_deploy_config(yaml_path)
|
||||
stage_0 = next(stage for stage in deploy_config.stages if stage.stage_id == 0)
|
||||
assert stage_0.engine_extras["attention_backend"] == "FLASHINFER"
|
||||
assert stage_0.engine_extras["attention_config"] == {"use_trtllm_attention": False}
|
||||
|
||||
|
||||
# ---- AC-3: Prompt Builder ----
|
||||
|
||||
@@ -90,6 +90,7 @@ class _ModelEngineOverrides(TypedDict, total=False):
|
||||
trust_remote_code: bool
|
||||
dtype: Any
|
||||
attention_backend: Any
|
||||
attention_config: Any
|
||||
moe_backend: str
|
||||
hf_overrides: Any
|
||||
limit_mm_per_prompt: dict[str, Any]
|
||||
@@ -321,6 +322,7 @@ class OmniStageModelConfig:
|
||||
trust_remote_code: bool = False
|
||||
dtype: Any = "auto"
|
||||
attention_backend: Any = None
|
||||
attention_config: Any = None
|
||||
moe_backend: str = "auto"
|
||||
hf_overrides: Any = None
|
||||
limit_mm_per_prompt: dict[str, Any] | None = None
|
||||
|
||||
@@ -44,6 +44,65 @@ vllm-omni serve bosonai/higgs-audio-v3-tts-4b \
|
||||
--deploy-config vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml
|
||||
```
|
||||
|
||||
## Why Stage 0 Pins Native FlashInfer
|
||||
|
||||
The three Stage 0 profiles set `attention_backend: FLASHINFER` and also set:
|
||||
|
||||
```yaml
|
||||
attention_config:
|
||||
use_trtllm_attention: false
|
||||
```
|
||||
|
||||
`FLASHINFER` names the overall vLLM attention backend. Starting with vLLM
|
||||
0.25, that backend can internally route Hopper decode to the TRT-LLM XQA
|
||||
kernel. The XQA eligibility check answers whether the kernel can run, not
|
||||
whether it is the fastest kernel for the current workload. In particular, the
|
||||
decode builder can select XQA statically when all of the following hold:
|
||||
|
||||
- the explicit `use_trtllm_attention` override is not `false`;
|
||||
- the device is Hopper/SM90 and the XQA cubin is available; and
|
||||
- the query-head count is divisible by the KV-head count.
|
||||
|
||||
Higgs has 32 query heads and 8 KV heads, so it passes the head-layout check.
|
||||
The static gate does not include a model dtype, KV-cache dtype, context-length,
|
||||
batch-shape, or measured-performance comparison. This matters because kernel
|
||||
support is not a performance guarantee.
|
||||
|
||||
Higgs also has an unusual output shape that must not be confused with its
|
||||
attention shape. One autoregressive audio step advances one token per request.
|
||||
The resulting hidden row is then projected into eight codebook distributions
|
||||
with shape `[1, 8, codebook_size]`. The eight codebooks therefore do not turn
|
||||
the attention query into `q_len=8`; at concurrency one, XQA still receives a
|
||||
single BF16 query row. At this small per-step shape, any XQA advantage for
|
||||
other KV dtypes, batch sizes, or context lengths does not necessarily apply.
|
||||
The difference is paid in every transformer layer and every generated audio
|
||||
step, so even a small per-call disadvantage accumulates in TTFP and RTF.
|
||||
|
||||
The upstream XQA integration also showed that the result is dtype-sensitive:
|
||||
its FP8 benchmark favored XQA, while its BF16 benchmark reported 21.16 ms mean
|
||||
TPOT for XQA versus 19.47 ms for FA3. That is not a direct native-FlashInfer
|
||||
comparison, but it demonstrates why SM90 compatibility alone is insufficient
|
||||
to choose the fastest BF16 kernel. See the
|
||||
[vLLM XQA benchmark](https://github.com/vllm-project/vllm/pull/43232#issuecomment-4501176335).
|
||||
|
||||
The end-to-end H800 A/B in
|
||||
[issue #5584](https://github.com/vllm-project/vllm-omni/issues/5584#issuecomment-5188700218)
|
||||
isolated the routing decision on the same vLLM 0.25 stack:
|
||||
|
||||
| Decode path | TTFP (ms) | RTF |
|
||||
|---|---:|---:|
|
||||
| XQA (automatic default) | 175.76 | 0.3405 |
|
||||
| Native FlashInfer (`use_trtllm_attention: false`) | 166.39 | 0.3233 |
|
||||
| vLLM 0.24 reference | 166.68 | 0.3209 |
|
||||
|
||||
This proves that XQA selection causes the observed local regression and that
|
||||
native FlashInfer recovers the pre-v0.25 range. It does not by itself identify
|
||||
one defective XQA instruction or memory transaction. Establishing that deeper
|
||||
kernel-level cause requires an attention-only microbenchmark and GPU profiler
|
||||
trace for the exact Higgs batch/context distribution. Until vLLM has a
|
||||
workload-aware selector or XQA is faster for this BF16 shape, the model-level
|
||||
pin is the deterministic choice.
|
||||
|
||||
## Notes
|
||||
|
||||
- Stage 1 remains `enforce_eager: true` in both profiles.
|
||||
|
||||
@@ -38,6 +38,11 @@ stages:
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: true
|
||||
attention_backend: FLASHINFER
|
||||
# Eight codebooks share one attention row, so decode remains BF16 q_len=1
|
||||
# per request. vLLM's SM90 XQA capability gate does not benchmark this
|
||||
# shape; keep the faster native path (README_higgs_audio_v3.md, #5584).
|
||||
attention_config:
|
||||
use_trtllm_attention: false
|
||||
trust_remote_code: true
|
||||
# Stage-0 prefix caching is on by default. The talker's HS opt-out and
|
||||
# deferred ``codes.audio`` mm-output write (see higgs_audio_v3_talker.py)
|
||||
|
||||
@@ -37,6 +37,11 @@ stages:
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: true
|
||||
attention_backend: FLASHINFER
|
||||
# Eight codebooks share one attention row, so decode remains BF16 q_len=1
|
||||
# per request. vLLM's SM90 XQA capability gate does not benchmark this
|
||||
# shape; keep the faster native path (README_higgs_audio_v3.md, #5584).
|
||||
attention_config:
|
||||
use_trtllm_attention: false
|
||||
trust_remote_code: true
|
||||
# Stage-0 prefix caching is on by default. The talker's HS opt-out and
|
||||
# deferred ``codes.audio`` mm-output write (see higgs_audio_v3_talker.py)
|
||||
|
||||
@@ -43,6 +43,11 @@ stages:
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: false
|
||||
attention_backend: FLASHINFER
|
||||
# Eight codebooks share one attention row, so decode remains BF16 q_len=1
|
||||
# per request. vLLM's SM90 XQA capability gate does not benchmark this
|
||||
# shape; keep the faster native path (README_higgs_audio_v3.md, #5584).
|
||||
attention_config:
|
||||
use_trtllm_attention: false
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: true
|
||||
async_scheduling: false
|
||||
|
||||
Reference in new issue
Block a user