[BugFix] Fix the issue of thinker requests being preempted, causing shape mismatch. (#3147)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-05-15 17:07:42 +08:00
1 parent 440c718d2a
commit e7ee5de09f
5 files changed
+65 -74

No files matched your search

@@ -990,19 +990,10 @@ class Qwen3OmniMoeForConditionalGeneration(
"""
embed = payload.get("embed", {})
meta = payload.get("meta", {})
ids = payload.get("ids", {})
cached_thinker_decode_embeds = embed.get("cached_decode", None)
thinker_decode_embed = embed.get("decode", None)
start_index = meta.get("num_processed_tokens", 0)
thinker_output_token_ids = ids.get("output", [])
if start_index >= len(thinker_output_token_ids) - 1:
# When the tokens output by the thinker are exhausted, an EOS token needs to be appended.
# Use the finished_flag to mark that all tokens output by thinker have been consumed.
if meta.get("eos_emitted", False):
return self.tts_pad_embed.to(device)
update_dict.setdefault("meta", {})["eos_emitted"] = True
return self.tts_eos_embed.to(device)
if cached_thinker_decode_embeds is not None and start_index < cached_thinker_decode_embeds.shape[0]:
cached_thinker_decode_embeds = cached_thinker_decode_embeds.to(device)
@@ -1011,10 +1002,20 @@ class Qwen3OmniMoeForConditionalGeneration(
thinker_decode_embed = thinker_decode_embed.to(device)
cached_thinker_decode_embeds = torch.cat([cached_thinker_decode_embeds, thinker_decode_embed], dim=0)
update_dict.setdefault("embed", {})["cached_decode"] = cached_thinker_decode_embeds
else:
elif thinker_decode_embed is not None:
thinker_embed = thinker_decode_embed
if thinker_embed.device != device:
thinker_embed = thinker_embed.to(device)
else:
# When the tokens output by the thinker are exhausted, an EOS token needs to be appended.
# Use the finished_flag to mark that all tokens output by thinker have been consumed.
if meta.get("eos_emitted", False):
return self.tts_pad_embed.to(device)
update_dict.setdefault("meta", {})["eos_emitted"] = True
return self.tts_eos_embed.to(device)
update_dict.setdefault("embed", {})["decode"] = None
return self.talker.text_projection(thinker_embed).to(device)
@@ -431,27 +431,28 @@ def thinker2talker_async_chunk(
payload.hidden_states.output = torch.cat(
(save_payload.get("hidden_states", {}).get("output"), payload.hidden_states.output), dim=0
)
prefill_shape = payload.embed.prefill.shape[0]
if not is_finished and prefill_shape <= len(prompt_token_ids):
transfer_manager.request_payload[request_id] = to_dict(payload)
return None
else:
output_token_ids = _ensure_list(request.output_token_ids)
if thinker_emb.shape[0] > 1:
logger.warning(
"Unexpected multiple embeddings in thinker2talker_async_chunk for chunk_id %d: "
"request_id %s, num_computed_tokens%d %s. Expected shape [1, D].",
chunk_id,
request_id,
request.num_computed_tokens,
thinker_emb.shape,
)
return None
meta = MetaStruct(finished=torch.tensor(is_finished, dtype=torch.bool))
if output_token_ids:
meta.override_keys = [("embed", "decode"), ("ids", "output")]
payload = OmniPayloadStruct(
meta=meta,
embed=EmbeddingsStruct(decode=thinker_emb.detach().cpu()),
ids=IdsStruct(output=output_token_ids),
speaker=speaker,
language=language,
)
else:
# When prefilling a chunked thinker, thinker_hidden_states needs to be updated.
payload = OmniPayloadStruct(
meta=meta,
embed=EmbeddingsStruct(prefill=thinker_emb.detach().cpu()),
hidden_states=HiddenStatesStruct(output=thinker_hid.detach().cpu()),
speaker=speaker,
language=language,
)
payload = OmniPayloadStruct(
meta=meta,
embed=EmbeddingsStruct(decode=thinker_emb.detach().cpu()),
speaker=speaker,
language=language,
)
return payload