[Bugfix] Fix Qwen3-Omni realtime playback interruption and conversation history (#7791)
Signed-off-by: psv666 <2693925048@qq.com>
This commit is contained in:
1 parent
d3396fc5fb
commit
2d0b03f717
13 files changed
+1292
-68
No files matched your search
@@ -213,16 +213,27 @@ class Qwen3OmniDuplexPlugin(DuplexModelPlugin):
|
||||
# The history accessor copies message dicts but preserves content identity.
|
||||
# Keep payloads privately, never put PCM into public conversation events.
|
||||
has_audio = bool(payload.get("audio"))
|
||||
last_content = history[-1].get("content") if history else None
|
||||
current_is_in_history = (
|
||||
has_audio
|
||||
and isinstance(last_content, list)
|
||||
and any(part.get("type") == "audio_url" for part in last_content)
|
||||
)
|
||||
# A committed audio item need not be last: camera/text items can arrive
|
||||
# before response.create. Search only the unanswered user turn so a new
|
||||
# payload can never overwrite an earlier, already answered utterance.
|
||||
current_contents = []
|
||||
if has_audio:
|
||||
for message in reversed(history):
|
||||
if message.get("role") != "user":
|
||||
break
|
||||
content = message.get("content")
|
||||
if isinstance(content, list) and any(part.get("type") == "audio_url" for part in content):
|
||||
current_contents.append(content)
|
||||
current_is_in_history = bool(current_contents)
|
||||
if current_is_in_history:
|
||||
content = history[-1].get("content")
|
||||
state.audio_history = [(key, value) for key, value in state.audio_history if key is not content]
|
||||
state.audio_history.append((content, payload))
|
||||
# The runner concatenates multiple commits made before a response.
|
||||
# Associate that single payload with all of its source audio items.
|
||||
state.audio_history = [
|
||||
(keys, value)
|
||||
for keys, value in state.audio_history
|
||||
if not any(key is content for key in keys for content in current_contents)
|
||||
]
|
||||
state.audio_history.append((tuple(current_contents), payload))
|
||||
# Reserve one of the four audio slots for the actual submitted input,
|
||||
# including when deferred input has no trailing user history entry.
|
||||
history_limit = 4 if current_is_in_history or not has_audio else 3
|
||||
@@ -230,33 +241,75 @@ class Qwen3OmniDuplexPlugin(DuplexModelPlugin):
|
||||
while len(retained) > 1 and sum(len(value.get("audio", "")) for _, value in retained) > 8 * 1024 * 1024:
|
||||
retained.pop(0)
|
||||
state.audio_history = retained
|
||||
# Match the official demo's unit of history: consecutive user items
|
||||
# followed by their assistant reply(s). Prune whole turns, before
|
||||
# formatting, so a camera item cannot rescue an orphaned answer.
|
||||
turns = []
|
||||
for index, message in enumerate(history):
|
||||
role = message.get("role")
|
||||
if not turns or role == "system" or (role == "user" and turns[-1][1][-1].get("role") != "user"):
|
||||
turns.append((index, []))
|
||||
turns[-1][1].append(message)
|
||||
|
||||
messages = []
|
||||
|
||||
def has_image(message):
|
||||
content = message.get("content")
|
||||
return isinstance(content, list) and any(p.get("type") == "image_url" for p in content)
|
||||
|
||||
selected = [m for m in history[:-16] if has_image(m)] + history[-16:]
|
||||
for message in selected:
|
||||
content = message.get("content")
|
||||
audio = next((value for key, value in retained if key is content), None)
|
||||
if audio is not None:
|
||||
messages.append({"role": "user", "audio_payload": audio})
|
||||
elif isinstance(content, list) and any(part.get("type") == "image_url" for part in content):
|
||||
messages.append(message)
|
||||
elif isinstance(content, str) and content:
|
||||
messages.append(message)
|
||||
elif message.get("role") == "user":
|
||||
# Do not keep an assistant answer after dropping its user audio.
|
||||
messages = [m for m in messages if has_image(m)]
|
||||
while messages and messages[0].get("role") == "assistant":
|
||||
messages.pop(0)
|
||||
for index, turn in turns:
|
||||
# Keep the existing recent-message window, rounded outwards to a
|
||||
# complete turn. Never retain an answer whose input was cut off.
|
||||
if index + len(turn) <= len(history) - 16:
|
||||
continue
|
||||
if turn[0].get("role") == "assistant":
|
||||
continue
|
||||
resolved = []
|
||||
included_audio = set()
|
||||
for message in turn:
|
||||
content = message.get("content")
|
||||
if message.get("role") == "user" and isinstance(content, list):
|
||||
if any(part.get("type") == "audio_url" for part in content):
|
||||
audio = next((value for keys, value in retained if any(key is content for key in keys)), None)
|
||||
if audio is None:
|
||||
break
|
||||
if id(audio) not in included_audio:
|
||||
resolved.append({"role": "user", "audio_payload": audio})
|
||||
included_audio.add(id(audio))
|
||||
# Preserve text/images in mixed source items as well.
|
||||
other_parts = [part for part in content if part.get("type") != "audio_url"]
|
||||
if other_parts:
|
||||
resolved.append({"role": "user", "content": other_parts})
|
||||
continue
|
||||
resolved.append(message)
|
||||
else:
|
||||
messages.extend(resolved)
|
||||
if has_audio and not current_is_in_history:
|
||||
# History is context, not a replacement for the current request.
|
||||
# Re-encoding only history here makes Qwen answer an old question.
|
||||
messages.append({"role": "user", "audio_payload": payload})
|
||||
return {**config, "qwen_messages": messages}
|
||||
|
||||
@staticmethod
|
||||
def format_history(messages):
|
||||
"""Merge user items as Qwen's official web_demo.format_history does.
|
||||
|
||||
This is a model-input view, not a mutation of addressable session items.
|
||||
Empty assistant messages remain boundaries for interrupted/unheard turns.
|
||||
"""
|
||||
formatted = []
|
||||
pending = []
|
||||
for message in messages:
|
||||
if message.get("role") == "user":
|
||||
content = message.get("content")
|
||||
pending.extend([{"type": "text", "text": content}] if isinstance(content, str) else content or [])
|
||||
continue
|
||||
if pending:
|
||||
formatted.append({"role": "user", "content": pending})
|
||||
pending = []
|
||||
formatted.append(message)
|
||||
if pending:
|
||||
# The official demo puts media before the final turn's instruction.
|
||||
media = [part for part in pending if part.get("type") != "text"]
|
||||
text = [part for part in pending if part.get("type") == "text"]
|
||||
formatted.append({"role": "user", "content": media + text})
|
||||
return formatted
|
||||
|
||||
@staticmethod
|
||||
def _trim_prompt_images(messages, images):
|
||||
"""Drop the oldest images, with their placeholders, down to the prompt limit.
|
||||
@@ -323,7 +376,7 @@ class Qwen3OmniDuplexPlugin(DuplexModelPlugin):
|
||||
raise DuplexRuntimeConfigError("Qwen generation requires committed audio or conversation history")
|
||||
audios = []
|
||||
images = []
|
||||
for index, message in enumerate(history):
|
||||
for message in history:
|
||||
if "audio_payload" not in message:
|
||||
content = message.get("content")
|
||||
if isinstance(content, list):
|
||||
@@ -343,10 +396,11 @@ class Qwen3OmniDuplexPlugin(DuplexModelPlugin):
|
||||
continue
|
||||
content, audio = self._audio_content(message["audio_payload"])
|
||||
audios.append((audio, 16000))
|
||||
if runtime_config.get("initial_user_text") and index == len(history) - 1:
|
||||
content.append({"type": "text", "text": runtime_config["initial_user_text"]})
|
||||
messages.append({"role": "user", "content": content})
|
||||
if payload.get("audio") and runtime_config.get("initial_user_text"):
|
||||
messages.append({"role": "user", "content": runtime_config["initial_user_text"]})
|
||||
messages, images = self._trim_prompt_images(messages, images)
|
||||
messages = self.format_history(messages)
|
||||
prompt = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
||||
mm = {"audio": audios} if audios else {}
|
||||
if images:
|
||||
|
||||
@@ -17,8 +17,9 @@ class QwenDuplexSessionState(DuplexModelSessionState):
|
||||
"""Mutable model-owned state of one Qwen duplex session (owned by the session runner)."""
|
||||
|
||||
audio_buffer: QwenPcmBuffer = field(default_factory=QwenPcmBuffer)
|
||||
# Keep content objects alongside payloads so identity survives history copies.
|
||||
audio_history: list[tuple[object, dict]] = field(default_factory=list)
|
||||
# Each payload can combine several audio commits from one pending turn.
|
||||
# Keep their content objects so associations survive shallow history copies.
|
||||
audio_history: list[tuple[tuple[object, ...], dict]] = field(default_factory=list)
|
||||
input_since_commit: bool = False
|
||||
speech_since_commit: bool = False
|
||||
context_locked: bool = False
|
||||
|
||||
Reference in new issue
Block a user