[Bugfix] fix precision issues of qwen3-omni when enable async_chunk without system prompt (#1288)

Signed-off-by: Rein Yang <ruiruyang2@gmail.com>
This commit is contained in:
rein yang authored and GitHub committed 2026-02-11 10:03:41 +08:00
1 parent dc8b28e0aa
commit 963b64e7ae
2 files changed
+55 -4

No files matched your search

+48
View File
@@ -22,6 +22,9 @@ from vllm.entrypoints.chat_utils import (
_postprocess_messages,
_ToolParser,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
class OmniAsyncMultiModalItemTracker(AsyncMultiModalItemTracker):
@@ -135,6 +138,48 @@ class OmniAsyncMultiModalContentParser(AsyncMultiModalContentParser):
await asyncio.to_thread(_cleanup_file_sync, temp_video_file_path)
def _ensure_system_prompt(
messages: list[ChatCompletionMessageParam],
model_config: ModelConfig,
) -> list[ChatCompletionMessageParam]:
"""
Ensure a system prompt exists for Qwen-Omni models to preserve precision of the model.
Args:
messages: List of chat messages
model_config: Model configuration
Returns:
Messages list with system prompt
"""
model_name = getattr(model_config, "model", "").lower()
hf_config = getattr(model_config, "hf_config", None)
architectures = getattr(hf_config, "architectures", []) if hf_config else []
is_qwen_omni = ("qwen" in model_name and "omni" in model_name) or any(
"qwen" in arch.lower() and "omni" in arch.lower() for arch in architectures
)
if not is_qwen_omni:
return messages
if messages and messages[0].get("role") == "system":
return messages
default_qwen_omni_system_prompt = (
"You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
"Group, capable of perceiving auditory and visual inputs, as well as "
"generating text and speech."
)
system_message: ChatCompletionMessageParam = {
"role": "system",
"content": default_qwen_omni_system_prompt,
}
logger.info(f"injecting system prompt {default_qwen_omni_system_prompt} for Qwen-Omni model")
return [system_message] + list(messages)
def parse_chat_messages_futures(
messages: list[ChatCompletionMessageParam],
model_config: ModelConfig,
@@ -153,6 +198,9 @@ def parse_chat_messages_futures(
Tuple of (conversation, mm_future) where mm_future resolves to
(mm_data, mm_uuids) when awaited.
"""
# auto-inject system prompt for Qwen-Omni models if missing
messages = _ensure_system_prompt(messages, model_config)
conversation: list[ConversationMessage] = []
mm_tracker = OmniAsyncMultiModalItemTracker(model_config)
@@ -589,6 +589,7 @@ class Qwen3OmniMoeForConditionalGeneration(
input_embeds = self.talker.embed_input_ids(input_ids)
span_len = input_ids.shape[0]
update_dict = {}
if span_len > 1:
# prefill
input_ids, input_embeds, update_dict = self.talker_preprocess_prefill(input_ids, input_embeds, **info_dict)
@@ -600,11 +601,12 @@ class Qwen3OmniMoeForConditionalGeneration(
update_dict["code_predictor_codes"] = code_predictor_codes
else:
# decode
if info_dict.get("num_processed_tokens", 0) < len(info_dict.get("thinker_input_ids", [])):
if not info_dict.get("decode_flag", False):
info_dict["num_processed_tokens"] = len(info_dict.get("thinker_input_ids", [])) + 1
update_dict["decode_flag"] = True
last_talker_hidden, text_step, update_dict = self.talker_preprocess_decode(
input_ids, input_embeds, **info_dict
input_ids, input_embeds, update_dict, **info_dict
)
update_dict["mtp_inputs"] = last_talker_hidden, text_step
@@ -874,8 +876,9 @@ class Qwen3OmniMoeForConditionalGeneration(
thinker_embed = thinker_embed[start_index : start_index + 1].to(device)
return self.talker.text_projection(thinker_embed).to(device)
def talker_preprocess_decode(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict):
update_dict: dict[str, dict] = {}
def talker_preprocess_decode(
self, input_ids: torch.Tensor, input_embeds: torch.Tensor, update_dict: dict, **info_dict: dict
):
last_talker_hidden = None
text_step = None
try: