[BugFix] Fix the issue of thinker requests being preempted, causing shape mismatch. (#3147)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
440c718d2a
commit
e7ee5de09f
5 files changed
+65
-74
No files matched your search
@@ -990,19 +990,10 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
"""
|
||||
embed = payload.get("embed", {})
|
||||
meta = payload.get("meta", {})
|
||||
ids = payload.get("ids", {})
|
||||
|
||||
cached_thinker_decode_embeds = embed.get("cached_decode", None)
|
||||
thinker_decode_embed = embed.get("decode", None)
|
||||
start_index = meta.get("num_processed_tokens", 0)
|
||||
thinker_output_token_ids = ids.get("output", [])
|
||||
if start_index >= len(thinker_output_token_ids) - 1:
|
||||
# When the tokens output by the thinker are exhausted, an EOS token needs to be appended.
|
||||
# Use the finished_flag to mark that all tokens output by thinker have been consumed.
|
||||
if meta.get("eos_emitted", False):
|
||||
return self.tts_pad_embed.to(device)
|
||||
update_dict.setdefault("meta", {})["eos_emitted"] = True
|
||||
return self.tts_eos_embed.to(device)
|
||||
|
||||
if cached_thinker_decode_embeds is not None and start_index < cached_thinker_decode_embeds.shape[0]:
|
||||
cached_thinker_decode_embeds = cached_thinker_decode_embeds.to(device)
|
||||
@@ -1011,10 +1002,20 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
thinker_decode_embed = thinker_decode_embed.to(device)
|
||||
cached_thinker_decode_embeds = torch.cat([cached_thinker_decode_embeds, thinker_decode_embed], dim=0)
|
||||
update_dict.setdefault("embed", {})["cached_decode"] = cached_thinker_decode_embeds
|
||||
else:
|
||||
|
||||
elif thinker_decode_embed is not None:
|
||||
thinker_embed = thinker_decode_embed
|
||||
if thinker_embed.device != device:
|
||||
thinker_embed = thinker_embed.to(device)
|
||||
|
||||
else:
|
||||
# When the tokens output by the thinker are exhausted, an EOS token needs to be appended.
|
||||
# Use the finished_flag to mark that all tokens output by thinker have been consumed.
|
||||
if meta.get("eos_emitted", False):
|
||||
return self.tts_pad_embed.to(device)
|
||||
update_dict.setdefault("meta", {})["eos_emitted"] = True
|
||||
return self.tts_eos_embed.to(device)
|
||||
|
||||
update_dict.setdefault("embed", {})["decode"] = None
|
||||
return self.talker.text_projection(thinker_embed).to(device)
|
||||
|
||||
|
||||
@@ -431,27 +431,28 @@ def thinker2talker_async_chunk(
|
||||
payload.hidden_states.output = torch.cat(
|
||||
(save_payload.get("hidden_states", {}).get("output"), payload.hidden_states.output), dim=0
|
||||
)
|
||||
prefill_shape = payload.embed.prefill.shape[0]
|
||||
if not is_finished and prefill_shape <= len(prompt_token_ids):
|
||||
transfer_manager.request_payload[request_id] = to_dict(payload)
|
||||
return None
|
||||
else:
|
||||
output_token_ids = _ensure_list(request.output_token_ids)
|
||||
if thinker_emb.shape[0] > 1:
|
||||
logger.warning(
|
||||
"Unexpected multiple embeddings in thinker2talker_async_chunk for chunk_id %d: "
|
||||
"request_id %s, num_computed_tokens%d %s. Expected shape [1, D].",
|
||||
chunk_id,
|
||||
request_id,
|
||||
request.num_computed_tokens,
|
||||
thinker_emb.shape,
|
||||
)
|
||||
return None
|
||||
meta = MetaStruct(finished=torch.tensor(is_finished, dtype=torch.bool))
|
||||
if output_token_ids:
|
||||
meta.override_keys = [("embed", "decode"), ("ids", "output")]
|
||||
payload = OmniPayloadStruct(
|
||||
meta=meta,
|
||||
embed=EmbeddingsStruct(decode=thinker_emb.detach().cpu()),
|
||||
ids=IdsStruct(output=output_token_ids),
|
||||
speaker=speaker,
|
||||
language=language,
|
||||
)
|
||||
else:
|
||||
# When prefilling a chunked thinker, thinker_hidden_states needs to be updated.
|
||||
payload = OmniPayloadStruct(
|
||||
meta=meta,
|
||||
embed=EmbeddingsStruct(prefill=thinker_emb.detach().cpu()),
|
||||
hidden_states=HiddenStatesStruct(output=thinker_hid.detach().cpu()),
|
||||
speaker=speaker,
|
||||
language=language,
|
||||
)
|
||||
payload = OmniPayloadStruct(
|
||||
meta=meta,
|
||||
embed=EmbeddingsStruct(decode=thinker_emb.detach().cpu()),
|
||||
speaker=speaker,
|
||||
language=language,
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user