From dbc0dd6d85272aab507c78b35f837d8651807087 Mon Sep 17 00:00:00 2001 From: Sy03 <1370724210@qq.com> Date: Thu, 13 Aug 2026 10:00:50 -0700 Subject: [PATCH] [Bugfix][Higgs-Audio-V3] Disable XQA decode (#6068) Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: Sysy <1370724210@qq.com> --- tests/config/test_omni_config.py | 1 + tests/engine/test_stage_engine_args.py | 2 + .../higgs_audio_v3/test_higgs_audio_v3.py | 8 ++- vllm_omni/config/omni_config.py | 2 + vllm_omni/deploy/README_higgs_audio_v3.md | 59 +++++++++++++++++++ vllm_omni/deploy/higgs_multimodal_qwen3.yaml | 5 ++ ...iggs_multimodal_qwen3_high_throughput.yaml | 5 ++ .../higgs_multimodal_qwen3_low_latency.yaml | 5 ++ 8 files changed, 86 insertions(+), 1 deletion(-) diff --git a/tests/config/test_omni_config.py b/tests/config/test_omni_config.py index 1fb32239d..cc08ed397 100644 --- a/tests/config/test_omni_config.py +++ b/tests/config/test_omni_config.py @@ -471,6 +471,7 @@ def test_sub_config_fields_match_structured_scopes(): "trust_remote_code", "dtype", "attention_backend", + "attention_config", "moe_backend", "hf_overrides", "limit_mm_per_prompt", diff --git a/tests/engine/test_stage_engine_args.py b/tests/engine/test_stage_engine_args.py index 75660cbc7..55ae43603 100644 --- a/tests/engine/test_stage_engine_args.py +++ b/tests/engine/test_stage_engine_args.py @@ -139,6 +139,7 @@ def _engine_arg_inputs(tmp_path: Path) -> tuple[PipelineConfig, DeployConfig, st engine_extras={ "model": str(stage_model), "attention_backend": "FLASHINFER", + "attention_config": {"use_trtllm_attention": False}, "hf_overrides": {"rope_scaling": {"factor": 2}}, "limit_mm_per_prompt": {"audio": 1}, "logits_processors": ["test.custom.LogitsProcessor"], @@ -229,6 +230,7 @@ def test_typed_llm_engine_args_preserve_legacy_adapter_behavior(tmp_path): assert thinker_args["omni_kv_config"] == {"need_send_cache": True} assert thinker_args["worker_cls"] == "test.custom.Worker" assert thinker_args["attention_backend"] == "FLASHINFER" + assert thinker_args["attention_config"] == {"use_trtllm_attention": False} assert thinker_args["trust_remote_code"] is True assert thinker_args["dtype"] == "float16" assert thinker_args["distributed_executor_backend"] == "mp" diff --git a/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py b/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py index 6d81f300a..77a32ad7b 100755 --- a/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py +++ b/tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py @@ -1326,7 +1326,9 @@ class TestRegistry: assert "higgs_multimodal_qwen3" in OMNI_PIPELINES - def test_deploy_yaml_exists(self): + def test_deploy_yaml_uses_native_flashinfer(self): + from vllm_omni.config.stage_config import load_deploy_config + for name in ( "higgs_multimodal_qwen3.yaml", "higgs_multimodal_qwen3_high_throughput.yaml", @@ -1334,6 +1336,10 @@ class TestRegistry: ): yaml_path = get_deploy_config_path(name) assert os.path.isfile(yaml_path), f"Deploy YAML not found at {yaml_path}" + deploy_config = load_deploy_config(yaml_path) + stage_0 = next(stage for stage in deploy_config.stages if stage.stage_id == 0) + assert stage_0.engine_extras["attention_backend"] == "FLASHINFER" + assert stage_0.engine_extras["attention_config"] == {"use_trtllm_attention": False} # ---- AC-3: Prompt Builder ---- diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 32ee2c826..8c4a933e2 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -90,6 +90,7 @@ class _ModelEngineOverrides(TypedDict, total=False): trust_remote_code: bool dtype: Any attention_backend: Any + attention_config: Any moe_backend: str hf_overrides: Any limit_mm_per_prompt: dict[str, Any] @@ -321,6 +322,7 @@ class OmniStageModelConfig: trust_remote_code: bool = False dtype: Any = "auto" attention_backend: Any = None + attention_config: Any = None moe_backend: str = "auto" hf_overrides: Any = None limit_mm_per_prompt: dict[str, Any] | None = None diff --git a/vllm_omni/deploy/README_higgs_audio_v3.md b/vllm_omni/deploy/README_higgs_audio_v3.md index ea1363069..44f7f75c7 100644 --- a/vllm_omni/deploy/README_higgs_audio_v3.md +++ b/vllm_omni/deploy/README_higgs_audio_v3.md @@ -44,6 +44,65 @@ vllm-omni serve bosonai/higgs-audio-v3-tts-4b \ --deploy-config vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml ``` +## Why Stage 0 Pins Native FlashInfer + +The three Stage 0 profiles set `attention_backend: FLASHINFER` and also set: + +```yaml +attention_config: + use_trtllm_attention: false +``` + +`FLASHINFER` names the overall vLLM attention backend. Starting with vLLM +0.25, that backend can internally route Hopper decode to the TRT-LLM XQA +kernel. The XQA eligibility check answers whether the kernel can run, not +whether it is the fastest kernel for the current workload. In particular, the +decode builder can select XQA statically when all of the following hold: + +- the explicit `use_trtllm_attention` override is not `false`; +- the device is Hopper/SM90 and the XQA cubin is available; and +- the query-head count is divisible by the KV-head count. + +Higgs has 32 query heads and 8 KV heads, so it passes the head-layout check. +The static gate does not include a model dtype, KV-cache dtype, context-length, +batch-shape, or measured-performance comparison. This matters because kernel +support is not a performance guarantee. + +Higgs also has an unusual output shape that must not be confused with its +attention shape. One autoregressive audio step advances one token per request. +The resulting hidden row is then projected into eight codebook distributions +with shape `[1, 8, codebook_size]`. The eight codebooks therefore do not turn +the attention query into `q_len=8`; at concurrency one, XQA still receives a +single BF16 query row. At this small per-step shape, any XQA advantage for +other KV dtypes, batch sizes, or context lengths does not necessarily apply. +The difference is paid in every transformer layer and every generated audio +step, so even a small per-call disadvantage accumulates in TTFP and RTF. + +The upstream XQA integration also showed that the result is dtype-sensitive: +its FP8 benchmark favored XQA, while its BF16 benchmark reported 21.16 ms mean +TPOT for XQA versus 19.47 ms for FA3. That is not a direct native-FlashInfer +comparison, but it demonstrates why SM90 compatibility alone is insufficient +to choose the fastest BF16 kernel. See the +[vLLM XQA benchmark](https://github.com/vllm-project/vllm/pull/43232#issuecomment-4501176335). + +The end-to-end H800 A/B in +[issue #5584](https://github.com/vllm-project/vllm-omni/issues/5584#issuecomment-5188700218) +isolated the routing decision on the same vLLM 0.25 stack: + +| Decode path | TTFP (ms) | RTF | +|---|---:|---:| +| XQA (automatic default) | 175.76 | 0.3405 | +| Native FlashInfer (`use_trtllm_attention: false`) | 166.39 | 0.3233 | +| vLLM 0.24 reference | 166.68 | 0.3209 | + +This proves that XQA selection causes the observed local regression and that +native FlashInfer recovers the pre-v0.25 range. It does not by itself identify +one defective XQA instruction or memory transaction. Establishing that deeper +kernel-level cause requires an attention-only microbenchmark and GPU profiler +trace for the exact Higgs batch/context distribution. Until vLLM has a +workload-aware selector or XQA is faster for this BF16 shape, the model-level +pin is the deterministic choice. + ## Notes - Stage 1 remains `enforce_eager: true` in both profiles. diff --git a/vllm_omni/deploy/higgs_multimodal_qwen3.yaml b/vllm_omni/deploy/higgs_multimodal_qwen3.yaml index da299415b..ae4e373cb 100644 --- a/vllm_omni/deploy/higgs_multimodal_qwen3.yaml +++ b/vllm_omni/deploy/higgs_multimodal_qwen3.yaml @@ -38,6 +38,11 @@ stages: gpu_memory_utilization: 0.6 enforce_eager: true attention_backend: FLASHINFER + # Eight codebooks share one attention row, so decode remains BF16 q_len=1 + # per request. vLLM's SM90 XQA capability gate does not benchmark this + # shape; keep the faster native path (README_higgs_audio_v3.md, #5584). + attention_config: + use_trtllm_attention: false trust_remote_code: true # Stage-0 prefix caching is on by default. The talker's HS opt-out and # deferred ``codes.audio`` mm-output write (see higgs_audio_v3_talker.py) diff --git a/vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml b/vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml index 8a751483c..bb37bdb4c 100644 --- a/vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml +++ b/vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml @@ -37,6 +37,11 @@ stages: gpu_memory_utilization: 0.6 enforce_eager: true attention_backend: FLASHINFER + # Eight codebooks share one attention row, so decode remains BF16 q_len=1 + # per request. vLLM's SM90 XQA capability gate does not benchmark this + # shape; keep the faster native path (README_higgs_audio_v3.md, #5584). + attention_config: + use_trtllm_attention: false trust_remote_code: true # Stage-0 prefix caching is on by default. The talker's HS opt-out and # deferred ``codes.audio`` mm-output write (see higgs_audio_v3_talker.py) diff --git a/vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml b/vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml index 0d6ae8218..13c6961e6 100644 --- a/vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml +++ b/vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml @@ -43,6 +43,11 @@ stages: gpu_memory_utilization: 0.6 enforce_eager: false attention_backend: FLASHINFER + # Eight codebooks share one attention row, so decode remains BF16 q_len=1 + # per request. vLLM's SM90 XQA capability gate does not benchmark this + # shape; keep the faster native path (README_higgs_audio_v3.md, #5584). + attention_config: + use_trtllm_attention: false trust_remote_code: true enable_prefix_caching: true async_scheduling: false