Support Qwen3 Omni thinker cuda graph (#523)
Signed-off-by: ZeldaHuang <hzm414167@alibaba-inc.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
1753a878e0
commit
1ae0c64fe0
5 files changed
+38
-30
No files matched your search
@@ -188,6 +188,12 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.thinker.make_empty_intermediate_tensors if self.model_stage == "thinker" else lambda: None
|
||||
)
|
||||
if self.model_stage == "thinker":
|
||||
self.tts_tokens = torch.tensor(
|
||||
[[self.config.tts_bos_token_id, self.config.tts_eos_token_id, self.config.tts_pad_token_id]],
|
||||
device=self._module_device(self.thinker),
|
||||
dtype=torch.long,
|
||||
)
|
||||
|
||||
# ==================== Device utilities ====================
|
||||
|
||||
@@ -302,28 +308,28 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
|
||||
# ========== Stage 1: Thinker ==========
|
||||
if self.model_stage == "thinker":
|
||||
# Normalize to batched inputs if needed
|
||||
_added_batch_dim = False
|
||||
if input_ids is not None and input_ids.ndim == 1:
|
||||
input_ids = input_ids.unsqueeze(0)
|
||||
_added_batch_dim = True
|
||||
if positions is not None and positions.ndim == 1:
|
||||
positions = positions.unsqueeze(0)
|
||||
_added_batch_dim = True
|
||||
if inputs_embeds is not None and inputs_embeds.ndim == 2:
|
||||
inputs_embeds = inputs_embeds.unsqueeze(0)
|
||||
_added_batch_dim = True
|
||||
|
||||
thinker_dev = self._module_device(self.thinker)
|
||||
if is_npu():
|
||||
# Normalize to batched inputs if needed
|
||||
_added_batch_dim = False
|
||||
if input_ids is not None and input_ids.ndim == 1:
|
||||
input_ids = input_ids.unsqueeze(0)
|
||||
_added_batch_dim = True
|
||||
if positions is not None and positions.ndim == 1:
|
||||
positions = positions.unsqueeze(0)
|
||||
_added_batch_dim = True
|
||||
if inputs_embeds is not None and inputs_embeds.ndim == 2:
|
||||
inputs_embeds = inputs_embeds.unsqueeze(0)
|
||||
_added_batch_dim = True
|
||||
|
||||
# Handle None input_ids
|
||||
if input_ids is None:
|
||||
input_ids = torch.zeros(
|
||||
inputs_embeds.shape[1],
|
||||
dtype=torch.long,
|
||||
device=thinker_dev,
|
||||
).unsqueeze(0)
|
||||
_added_batch_dim = True
|
||||
# Handle None input_ids
|
||||
if input_ids is None:
|
||||
input_ids = torch.zeros(
|
||||
inputs_embeds.shape[1],
|
||||
dtype=torch.long,
|
||||
device=thinker_dev,
|
||||
).unsqueeze(0)
|
||||
_added_batch_dim = True
|
||||
|
||||
# Move to thinker device
|
||||
if input_ids is not None and input_ids.device != thinker_dev:
|
||||
@@ -365,12 +371,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
# These will later be projected into talker text space by the talker stage.
|
||||
multimodal_outputs = captured_layer_dict if captured_layer_dict is not None else {}
|
||||
try:
|
||||
tts_tokens = torch.tensor(
|
||||
[[self.config.tts_bos_token_id, self.config.tts_eos_token_id, self.config.tts_pad_token_id]],
|
||||
device=self._module_device(self.thinker),
|
||||
dtype=torch.long,
|
||||
)
|
||||
thinker_tts_embeds = self.thinker.embed_input_ids(tts_tokens) # [1,3,thinker_hidden]
|
||||
thinker_tts_embeds = self.thinker.embed_input_ids(self.tts_tokens) # [1,3,thinker_hidden]
|
||||
if (
|
||||
isinstance(thinker_tts_embeds, torch.Tensor)
|
||||
and thinker_tts_embeds.ndim == 3
|
||||
|
||||
Reference in new issue
Block a user