Merge remote-tracking branch 'origin/main' into dev/vllm-align
This commit is contained in:
commit
663976b8ca
2 files changed
+29
-43
No files matched your search
@@ -1588,20 +1588,6 @@ class HunyuanFusedMoE(SharedFusedMoE):
|
|||||||
self._init_hook_handle.remove()
|
self._init_hook_handle.remove()
|
||||||
|
|
||||||
def forward(self, hidden_states, router_logits):
|
def forward(self, hidden_states, router_logits):
|
||||||
from vllm.model_executor.layers.fused_moe.layer import get_forward_context
|
|
||||||
|
|
||||||
ctx = get_forward_context()
|
|
||||||
if not ctx.remaining_moe_layers:
|
|
||||||
import re
|
|
||||||
|
|
||||||
moe_names = [name for name in ctx.no_compile_layers.keys() if ".mlp.experts" in name]
|
|
||||||
|
|
||||||
def get_layer_num(name):
|
|
||||||
match = re.search(r"layers\.(\d+)\.mlp", name)
|
|
||||||
return int(match.group(1)) if match else -1
|
|
||||||
|
|
||||||
moe_names.sort(key=get_layer_num, reverse=True)
|
|
||||||
ctx.remaining_moe_layers.extend(moe_names)
|
|
||||||
return super().forward(hidden_states, router_logits)
|
return super().forward(hidden_states, router_logits)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -804,17 +804,14 @@ class HunyuanImage3Pipeline(HunyuanImage3PreTrainedModel, GenerationMixin):
|
|||||||
)
|
)
|
||||||
kwargs["num_image_tokens"] = num_image_tokens
|
kwargs["num_image_tokens"] = num_image_tokens
|
||||||
# 50 and 5.0 hard code
|
# 50 and 5.0 hard code
|
||||||
from vllm.forward_context import set_forward_context
|
results = self.pipeline(
|
||||||
|
batch_size=len(batch_gen_image_info),
|
||||||
with set_forward_context(None, self.vllm_config):
|
image_size=[batch_gen_image_info[0].image_height, batch_gen_image_info[0].image_width],
|
||||||
results = self.pipeline(
|
num_inference_steps=kwargs.get("num_inference_steps", 50),
|
||||||
batch_size=len(batch_gen_image_info),
|
guidance_scale=kwargs.get("guidance_scale", 5.0),
|
||||||
image_size=[batch_gen_image_info[0].image_height, batch_gen_image_info[0].image_width],
|
generator=generator,
|
||||||
num_inference_steps=kwargs.get("num_inference_steps", 50),
|
model_kwargs=kwargs,
|
||||||
guidance_scale=kwargs.get("guidance_scale", 5.0),
|
)
|
||||||
generator=generator,
|
|
||||||
model_kwargs=kwargs,
|
|
||||||
)
|
|
||||||
samples = results[0]
|
samples = results[0]
|
||||||
return samples
|
return samples
|
||||||
|
|
||||||
@@ -929,24 +926,27 @@ class HunyuanImage3Pipeline(HunyuanImage3PreTrainedModel, GenerationMixin):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
|
# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
|
||||||
outputs = self.model(
|
from vllm.forward_context import set_forward_context
|
||||||
input_ids=input_ids,
|
|
||||||
attention_mask=attention_mask,
|
with set_forward_context(None, self.vllm_config):
|
||||||
position_ids=position_ids,
|
outputs = self.model(
|
||||||
past_key_values=past_key_values,
|
input_ids=input_ids,
|
||||||
inputs_embeds=inputs_embeds,
|
attention_mask=attention_mask,
|
||||||
use_cache=use_cache,
|
position_ids=position_ids,
|
||||||
output_attentions=output_attentions,
|
past_key_values=past_key_values,
|
||||||
output_hidden_states=output_hidden_states,
|
inputs_embeds=inputs_embeds,
|
||||||
return_dict=return_dict,
|
use_cache=use_cache,
|
||||||
custom_pos_emb=custom_pos_emb,
|
output_attentions=output_attentions,
|
||||||
mode=mode,
|
output_hidden_states=output_hidden_states,
|
||||||
first_step=first_step,
|
return_dict=return_dict,
|
||||||
query_lens=query_lens,
|
custom_pos_emb=custom_pos_emb,
|
||||||
seq_lens=seq_lens,
|
mode=mode,
|
||||||
num_image_tokens=num_image_tokens,
|
first_step=first_step,
|
||||||
gen_timestep_scatter_index=gen_timestep_scatter_index,
|
query_lens=query_lens,
|
||||||
)
|
seq_lens=seq_lens,
|
||||||
|
num_image_tokens=num_image_tokens,
|
||||||
|
gen_timestep_scatter_index=gen_timestep_scatter_index,
|
||||||
|
)
|
||||||
hidden_states = outputs[0]
|
hidden_states = outputs[0]
|
||||||
|
|
||||||
if mode == "gen_text":
|
if mode == "gen_text":
|
||||||
|
|||||||
Reference in new issue
Block a user