[Perf] Optimize Higgs Audio v3 serving (#4204)

Signed-off-by: Sy03 <1370724210@qq.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-06-11 13:29:31 -07:00
1 parent eb964dcec5
commit e27fd46ab9
21 files changed
+1976 -734

No files matched your search

+57 -17
View File
@@ -57,6 +57,16 @@ class OmniGPUModelRunner(GPUModelRunner):
# The Omni tensor prefix cache will be allocated
# when we initialize the metadata builders if enabled
self.omni_prefix_cache = None
self._sampled_token_ids_cpu_override = None
self._omni_query_start_loc_model_kwarg = False
def _to_list(self, sampled_token_ids: torch.Tensor) -> list[list[int]]:
override_fn = self._sampled_token_ids_cpu_override
if callable(override_fn):
sampled = override_fn(sampled_token_ids)
if sampled is not None:
return sampled
return super()._to_list(sampled_token_ids)
def _omni_routed_experts_d2h(self, scheduler_output) -> None:
"""Issue routed-experts D2H copy matching upstream GPUModelRunner pattern.
@@ -105,14 +115,13 @@ class OmniGPUModelRunner(GPUModelRunner):
)
def initialize_metadata_builders(self, kv_cache_config, kernel_block_sizes):
"""Override to fix scheduler_metadata buffer size for FA3 + CUDA graph.
"""Initialize metadata builders and keep FA3 graph metadata buffers sized.
The upstream FlashAttentionMetadataBuilder pre-allocates
scheduler_metadata with (max_num_seqs + 1) entries, but FA3's
get_scheduler_metadata() can return up to
(max_num_seqs * max_num_splits + 1) entries, causing a RuntimeError
during CUDA graph capture. After calling the parent implementation
we resize any too-small buffers.
FlashAttentionMetadataBuilder can pre-allocate scheduler_metadata for
only max_num_seqs + 1 entries while FA3 with split scheduling may need
max_num_seqs * max_num_splits + 1 entries during CUDA graph capture.
This runner is shared across Omni models, so preserve the existing
workaround for non-Higgs models that still use FA3.
"""
super().initialize_metadata_builders(kv_cache_config, kernel_block_sizes)
@@ -143,6 +152,14 @@ class OmniGPUModelRunner(GPUModelRunner):
@instrument(span_name="Loading (GPU)")
def load_model(self, *args, **kwargs) -> None:
super().load_model(*args, **kwargs)
model = getattr(self, "model", None)
override_fn = None
if bool(getattr(model, "supports_sampled_token_ids_cpu_override", False)):
candidate = getattr(model, "consume_sampled_token_ids_cpu_override", None)
if callable(candidate):
override_fn = candidate
self._sampled_token_ids_cpu_override = override_fn
self._omni_query_start_loc_model_kwarg = bool(getattr(model, "supports_omni_query_start_loc", False))
self._maybe_enable_output_token_ids_for_model_sampler()
self._init_talker_mtp()
self._prewarm_attention_capture_workspaces()
@@ -1295,6 +1312,13 @@ class OmniGPUModelRunner(GPUModelRunner):
traceback.print_exc()
if self._omni_query_start_loc_model_kwarg:
try:
num_reqs = len(self.input_batch.req_ids)
model_kwargs_extra["omni_query_start_loc"] = self.query_start_loc.gpu[: num_reqs + 1]
except Exception as e:
logger.debug("[OMNI] Failed to attach query_start_loc: %s", e)
if getattr(self.model_config, "has_sampling_extra_args", False):
extra_args_list: list[dict] = []
for req_id in self.input_batch.req_ids:
@@ -1320,21 +1344,29 @@ class OmniGPUModelRunner(GPUModelRunner):
# execute the custom postprocess function
# TODO(Peiqi): do we have a more elegant way to do this?
if hasattr(self.model, "has_postprocess") and self.model.has_postprocess:
postprocess_uses_hidden_states = getattr(self.model, "postprocess_uses_hidden_states", True)
postprocess_uses_multimodal_outputs = getattr(self.model, "postprocess_uses_multimodal_outputs", True)
postprocess_uses_req_infos = getattr(self.model, "postprocess_uses_req_infos", True)
for req_index, req_id in enumerate(self.input_batch.req_ids):
if req_ids_filter is not None and req_id not in req_ids_filter:
continue
req_infos = self.model_intermediate_buffer.get(req_id, {})
if combined_hidden_states:
# Combined hidden states contains all hidden states for every request
hidden_states_slice = combined_hidden_states[req_id]
req_infos = self.model_intermediate_buffer.get(req_id, {}) if postprocess_uses_req_infos else {}
if postprocess_uses_hidden_states:
if combined_hidden_states:
# Combined hidden states contains all hidden states for every request
hidden_states_slice = combined_hidden_states[req_id]
else:
start_offset = int(self.query_start_loc.cpu[req_index])
sched_tokens = int(num_scheduled_tokens_np[req_index])
s, e = start_offset, start_offset + sched_tokens
# only consider to store data into update dict.
hidden_states_slice = hidden_states[s:e]
else:
start_offset = int(self.query_start_loc.cpu[req_index])
sched_tokens = int(num_scheduled_tokens_np[req_index])
s, e = start_offset, start_offset + sched_tokens
# only consider to store data into update dict.
hidden_states_slice = hidden_states[s:e]
hidden_states_slice = hidden_states
if combined_multimodal_outputs:
if not postprocess_uses_multimodal_outputs:
mm_out = None
elif combined_multimodal_outputs:
# NOTE this is a bit ugly, but the mm data is structured as a list of
# keys mapping to request IDs, and if enabled, we will always have all
# request IDs in every subdict, including for cache misses.
@@ -1807,6 +1839,14 @@ class OmniGPUModelRunner(GPUModelRunner):
):
"""Inject omni-specific kwargs into forward and cache model output"""
model_kwargs_extra = self._build_model_kwargs_extra()
update_decode_metadata = getattr(self.model, "update_decode_step_metadata", None)
if getattr(self.model, "supports_omni_decode_step_metadata", False) and callable(update_decode_metadata):
update_decode_metadata(
input_ids=input_ids,
positions=positions,
inputs_embeds=inputs_embeds,
omni_query_start_loc=model_kwargs_extra.get("omni_query_start_loc"),
)
model_output = super()._model_forward(
input_ids=input_ids,