Support Qwen3 Omni thinker cuda graph (#523)

Signed-off-by: ZeldaHuang <hzm414167@alibaba-inc.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Ziming HuangandHongsheng Liu authored and GitHub committed 2025-12-30 23:46:13 +13:00
1 parent 1753a878e0
commit 1ae0c64fe0
5 files changed
+38 -30

No files matched your search

@@ -188,6 +188,12 @@ class Qwen3OmniMoeForConditionalGeneration(
self.make_empty_intermediate_tensors = (
self.thinker.make_empty_intermediate_tensors if self.model_stage == "thinker" else lambda: None
)
if self.model_stage == "thinker":
self.tts_tokens = torch.tensor(
[[self.config.tts_bos_token_id, self.config.tts_eos_token_id, self.config.tts_pad_token_id]],
device=self._module_device(self.thinker),
dtype=torch.long,
)
# ==================== Device utilities ====================
@@ -302,28 +308,28 @@ class Qwen3OmniMoeForConditionalGeneration(
# ========== Stage 1: Thinker ==========
if self.model_stage == "thinker":
# Normalize to batched inputs if needed
_added_batch_dim = False
if input_ids is not None and input_ids.ndim == 1:
input_ids = input_ids.unsqueeze(0)
_added_batch_dim = True
if positions is not None and positions.ndim == 1:
positions = positions.unsqueeze(0)
_added_batch_dim = True
if inputs_embeds is not None and inputs_embeds.ndim == 2:
inputs_embeds = inputs_embeds.unsqueeze(0)
_added_batch_dim = True
thinker_dev = self._module_device(self.thinker)
if is_npu():
# Normalize to batched inputs if needed
_added_batch_dim = False
if input_ids is not None and input_ids.ndim == 1:
input_ids = input_ids.unsqueeze(0)
_added_batch_dim = True
if positions is not None and positions.ndim == 1:
positions = positions.unsqueeze(0)
_added_batch_dim = True
if inputs_embeds is not None and inputs_embeds.ndim == 2:
inputs_embeds = inputs_embeds.unsqueeze(0)
_added_batch_dim = True
# Handle None input_ids
if input_ids is None:
input_ids = torch.zeros(
inputs_embeds.shape[1],
dtype=torch.long,
device=thinker_dev,
).unsqueeze(0)
_added_batch_dim = True
# Handle None input_ids
if input_ids is None:
input_ids = torch.zeros(
inputs_embeds.shape[1],
dtype=torch.long,
device=thinker_dev,
).unsqueeze(0)
_added_batch_dim = True
# Move to thinker device
if input_ids is not None and input_ids.device != thinker_dev:
@@ -365,12 +371,7 @@ class Qwen3OmniMoeForConditionalGeneration(
# These will later be projected into talker text space by the talker stage.
multimodal_outputs = captured_layer_dict if captured_layer_dict is not None else {}
try:
tts_tokens = torch.tensor(
[[self.config.tts_bos_token_id, self.config.tts_eos_token_id, self.config.tts_pad_token_id]],
device=self._module_device(self.thinker),
dtype=torch.long,
)
thinker_tts_embeds = self.thinker.embed_input_ids(tts_tokens) # [1,3,thinker_hidden]
thinker_tts_embeds = self.thinker.embed_input_ids(self.tts_tokens) # [1,3,thinker_hidden]
if (
isinstance(thinker_tts_embeds, torch.Tensor)
and thinker_tts_embeds.ndim == 3