[Frontend] Add a shared realtime web UI for MiniCPM-o and Qwen3-Omni (#7222) (#7585)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-09-17 23:32:31 +08:00
1 parent eb1abbadf8
commit 882bf8bc95
53 files changed
+3438 -633

No files matched your search

@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
@@ -0,0 +1,103 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Transactional PCM buffering for committed Qwen speech turns."""
from __future__ import annotations
from dataclasses import dataclass, field
import pybase64 as base64
from vllm_omni.engine.duplex.plugin import PcmAppendBuffer, PcmAppendReservation
@dataclass
class Reservation(PcmAppendReservation):
owner: QwenPcmBuffer
operation_id: str
payload: dict[str, object] | None
raw: bytes
_active: bool = True
@property
def active(self):
return self._active
@property
def byte_count(self):
return len(self.raw)
def commit(self):
if self._active:
self._active = False
self.owner.reservations.remove(self)
def rollback(self):
if self._active:
self.owner.buffer[:0] = self.raw
self.commit()
@dataclass
class QwenPcmBuffer(PcmAppendBuffer):
buffer: bytearray = field(default_factory=bytearray)
reservations: list[Reservation] = field(default_factory=list)
@property
def pending_byte_count(self):
return len(self.buffer)
def clear(self):
for reservation in self.reservations:
reservation._active = False
self.reservations.clear()
self.buffer.clear()
def clear_force_listen(self):
pass
def has_pending(self):
return bool(self.buffer)
def has_reserved(self):
return bool(self.reservations)
def prepare_append(self, payload, *, operation_id, chunk_period_ms, allow_emit):
if payload.get("format") != "pcm_f32le" or payload.get("sample_rate_hz") != 16000:
raise ValueError("Qwen duplex requires normalized 16 kHz pcm_f32le")
if payload.get("video_frames"):
# Qwen answers whole turns, so it has no audio unit for a frame
# track to interleave against. Images arrive the OpenAI way, as
# input_image conversation items, and stay until deleted.
raise ValueError(
"Qwen duplex does not take video_frames on an audio append; "
"send images as conversation.item.create items with input_image content"
)
raw = base64.b64decode(payload.get("audio", ""), validate=True)
if len(raw) % 4:
raise ValueError("Invalid float32 PCM length")
self.buffer.extend(raw)
# Qwen inference starts on commit, never on an individual audio chunk.
return None
def prepare_commit(self, *, operation_id, chunk_period_ms):
raw = bytes(self.buffer)
payload = (
{
"audio": base64.b64encode(raw).decode(),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
"is_speech": True,
}
if raw
else None
)
self.buffer.clear()
reservation = Reservation(self, operation_id, payload, raw)
self.reservations.append(reservation)
return reservation
def flush(self, *, chunk_period_ms):
reservation = self.prepare_commit(operation_id="flush", chunk_period_ms=chunk_period_ms)
reservation.commit()
return reservation.payload
@@ -0,0 +1,390 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Engine-owned VAD/commit turns with concurrent input and interruptible output.
Qwen3-Omni is a turn model: this plugin does not advertise native streaming KV
or model-driven turn decisions. Each committed utterance is a fresh request.
"""
from __future__ import annotations
import asyncio
import copy
import hashlib
import io
import json
import os
from collections.abc import Mapping
from pathlib import Path
import numpy as np
import pybase64 as base64
from PIL import Image
from vllm.logger import init_logger
from vllm.sampling_params import RequestOutputKind
from vllm_omni.engine.duplex.config import DuplexCapabilities
from vllm_omni.engine.duplex.contracts import DuplexAppendPlan, duplex_resource_request_belongs_to_session
from vllm_omni.engine.duplex.plugin import DuplexDataPlane, DuplexModelPlugin, DuplexRuntimeConfigError
from vllm_omni.model_executor.models.qwen3_omni.duplex.session import QwenDuplexSessionState
logger = init_logger(__name__)
#: Mirrors ``limit_mm_per_prompt.image`` in ``deploy/qwen3_omni_duplex.yaml``.
#: The session already refuses a ninth stored image, but that budget lives in
#: the engine and this one in the deploy config; the prompt is the only place
#: that can keep them from drifting apart into a rejected submission.
MAX_PROMPT_IMAGES = 8
class QwenDataPlane(DuplexDataPlane):
def __init__(self, encode_audio):
self.encode_audio = encode_audio
self.terminal: set[str] = set()
def begin_request(self, request_id):
self.terminal.discard(request_id)
def is_terminal(self, request_id):
return request_id in self.terminal
def mark_terminal(self, request_id):
self.terminal.add(request_id)
def close_stream(self, request_id):
pass
def close_session(self, session_id, *, active_request_id=None):
self.terminal = {
rid for rid in self.terminal if not duplex_resource_request_belongs_to_session(rid, session_id)
}
def project(self, result, *, context=None):
if not isinstance(result, dict):
return
context = context or {}
for output in result.get("data_plane_outputs", []):
rid = output.request_id
if self.is_terminal(rid):
continue
completion = next(iter(getattr(output, "outputs", ()) or ()), None)
stage_id = getattr(output, "stage_id", 0)
text = getattr(completion, "text", "") if stage_id == 0 else ""
mm = getattr(output, "multimodal_output", None)
if not isinstance(mm, Mapping) or "audio" not in mm:
mm = getattr(completion, "multimodal_output", {}) or {}
audio = mm.get("audio")
encoded = None
duration_ms = 0
rate = mm.get("sr", mm.get("sample_rate", mm.get("sample_rate_hz", 24000)))
if isinstance(rate, (list, tuple)):
rate = next((value for value in rate if value is not None), 24000)
rate = int(rate)
if rate <= 0:
raise ValueError("Invalid output audio sample rate")
if audio is not None and "audio" in context.get("modalities", ()):
chunks = audio if isinstance(audio, (list, tuple)) else [audio]
arrays = [
np.asarray(
chunk.detach().float().cpu().numpy() if hasattr(chunk, "detach") else chunk, dtype=np.float32
).reshape(-1)
for chunk in chunks
]
audio = np.concatenate(arrays) if arrays else np.empty(0, dtype=np.float32)
duration_ms = audio.size * 1000 / rate
if audio.size:
encoded = self.encode_audio(
audio, rate, context.get("response_format", "pcm16"), context.get("speed")
)
yield {
"supported": True,
"data_plane_request_id": rid,
"text": text or "",
"audio_data": encoded or "",
"audio_format": context.get("response_format", "pcm16"),
"audio_duration_ms": duration_ms,
# Thinker text and codec chunks are independent streams. Only
# the final audio boundary can conservatively attest all text.
"audio_text_mark": False,
"text_requires_complete_audio": "audio" in context.get("modalities", ()),
"audio_complete": bool(output.finished) and stage_id == 2,
"sample_rate_hz": rate,
"end_of_turn": bool(output.finished),
"model_turn_id": context.get("active_response_turn_id")
if context.get("active_response_turn_id") is not None
else context.get("turn_id", 0),
"runtime_impl": "qwen3_commit",
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
}
class Qwen3OmniDuplexPlugin(DuplexModelPlugin):
plugin_id = "qwen3-omni"
projects_intermediate_outputs = True
def __init__(self, encode_audio):
self.data_plane = QwenDataPlane(encode_audio)
self.processor = None
self._processor_lock = asyncio.Lock()
def capabilities(self, *, max_sessions):
return DuplexCapabilities(
supports_model_native_turn_policy=False,
supports_input_append=True,
supports_barge_in=True,
supports_replace_latest_chunk=False,
supports_reencode_context=True,
supports_turn_commit_only=True,
supports_core_resumable_request=False,
supports_independent_io_streams=True,
supports_realtime_endpoint=True,
supports_multi_session=max_sessions > 1,
supports_multi_session_same_replica=max_sessions > 1,
supports_session_lease=True,
supports_session_resume=True,
session_admission_mode="engine_managed",
supports_audio_truncate=True,
supports_chat_completions=True,
supports_image_input=True,
supports_text_only_turn=True,
adapter_patterns=["turn_commit"],
signal_sources=["client_event", "server_policy"],
stage_handoff_transport="scheduler_data_plane",
target_barge_in_latency_ms=None,
)
def create_session_state(self):
return QwenDuplexSessionState()
def validate_client_extra_body(self, extra_body):
if not isinstance(extra_body, Mapping):
return
if extra_body.get("auto_response") or extra_body.get("full_duplex"):
raise DuplexRuntimeConfigError(
"Qwen requires server VAD or explicit commits; native auto_response is unsupported"
)
if extra_body.get("realtime_tools"):
raise DuplexRuntimeConfigError("Qwen duplex tool calls are not implemented")
async def prepare_runtime_config(self, config, *, model_config):
self.validate_client_extra_body(config.extra_body)
if config.ref_audio:
raise DuplexRuntimeConfigError("Qwen duplex does not support reference voice audio")
if model_config is None and self.processor is None:
raise DuplexRuntimeConfigError("Qwen duplex requires the stage-0 model configuration")
async with self._processor_lock:
if self.processor is None:
from transformers import AutoProcessor
self.processor = await asyncio.to_thread(
AutoProcessor.from_pretrained,
model_config.model,
trust_remote_code=model_config.trust_remote_code,
revision=getattr(model_config, "revision", None),
)
return self.runtime_config_for_update(config, {})
def runtime_config_for_update(self, config, current):
self.validate_client_extra_body(config.extra_body)
if config.ref_audio:
raise DuplexRuntimeConfigError("Qwen duplex does not support reference voice audio")
return {
"instructions": config.instructions,
"initial_user_text": config.initial_user_text,
"temperature": config.temperature,
"max_tokens": config.max_tokens,
}
def configure_sampling_params(self, *, runtime_config, defaults):
params = copy.deepcopy(defaults)
for value in params:
if value is not None:
value.output_kind = RequestOutputKind.DELTA
if params and params[0] is not None:
if runtime_config.get("temperature") is not None:
params[0].temperature = float(runtime_config["temperature"])
if isinstance(runtime_config.get("max_tokens"), int):
params[0].max_tokens = runtime_config["max_tokens"]
return params
def prepare_prompt_config(self, config, *, state, payload):
history = config.get("conversation", [])
# The history accessor copies message dicts but preserves content identity.
# Keep payloads privately, never put PCM into public conversation events.
has_audio = bool(payload.get("audio"))
last_content = history[-1].get("content") if history else None
current_is_in_history = (
has_audio
and isinstance(last_content, list)
and any(part.get("type") == "audio_url" for part in last_content)
)
if current_is_in_history:
content = history[-1].get("content")
state.audio_history = [(key, value) for key, value in state.audio_history if key is not content]
state.audio_history.append((content, payload))
# Reserve one of the four audio slots for the actual submitted input,
# including when deferred input has no trailing user history entry.
history_limit = 4 if current_is_in_history or not has_audio else 3
retained = state.audio_history[-history_limit:]
while len(retained) > 1 and sum(len(value.get("audio", "")) for _, value in retained) > 8 * 1024 * 1024:
retained.pop(0)
state.audio_history = retained
messages = []
def has_image(message):
content = message.get("content")
return isinstance(content, list) and any(p.get("type") == "image_url" for p in content)
selected = [m for m in history[:-16] if has_image(m)] + history[-16:]
for message in selected:
content = message.get("content")
audio = next((value for key, value in retained if key is content), None)
if audio is not None:
messages.append({"role": "user", "audio_payload": audio})
elif isinstance(content, list) and any(part.get("type") == "image_url" for part in content):
messages.append(message)
elif isinstance(content, str) and content:
messages.append(message)
elif message.get("role") == "user":
# Do not keep an assistant answer after dropping its user audio.
messages = [m for m in messages if has_image(m)]
while messages and messages[0].get("role") == "assistant":
messages.pop(0)
if has_audio and not current_is_in_history:
# History is context, not a replacement for the current request.
# Re-encoding only history here makes Qwen answer an old question.
messages.append({"role": "user", "audio_payload": payload})
return {**config, "qwen_messages": messages}
@staticmethod
def _trim_prompt_images(messages, images):
"""Drop the oldest images, with their placeholders, down to the prompt limit.
The processor pairs ``{"type": "image"}`` parts with ``mm["image"]``
positionally, and the placeholders read across *messages* in the same
order as *images*, so a dropped image has to take its placeholder with
it. Oldest first keeps what the client shared most recently.
"""
excess = len(images) - MAX_PROMPT_IMAGES
if excess <= 0:
return messages, images
logger.warning(
"Qwen duplex prompt carries %d images; dropping the %d oldest to stay within %d",
len(images),
excess,
MAX_PROMPT_IMAGES,
)
remaining = excess
for message in messages:
if remaining <= 0:
break
content = message.get("content")
if not isinstance(content, list):
continue
kept = []
for part in content:
if remaining > 0 and isinstance(part, dict) and part.get("type") == "image":
remaining -= 1
continue
kept.append(part)
message["content"] = kept
return [message for message in messages if message.get("content") != []], images[excess:]
@staticmethod
def _audio_content(payload):
raw = base64.b64decode(payload["audio"], validate=True)
audio = np.frombuffer(raw, dtype="<f4").copy()
if not audio.size or not np.isfinite(audio).all():
raise DuplexRuntimeConfigError("Audio must contain finite PCM samples")
return [{"type": "audio"}], audio
async def prepare_append_plan(self, **kwargs):
# History mutation has already happened on the session loop. The worker
# owns this snapshot and never reads or mutates live session state.
snapshot = copy.deepcopy(kwargs)
return await asyncio.to_thread(self.plan_append, **snapshot)
def plan_append(
self, *, request_id, fence, session_config, runtime_config, seq, turn_seq, payload, final, sampling_params
):
if not final or not isinstance(payload, dict):
raise DuplexRuntimeConfigError("Qwen generation requires a committed audio turn")
messages = []
if runtime_config.get("instructions"):
messages.append({"role": "system", "content": runtime_config["instructions"]})
history = session_config.get("qwen_messages")
if history is None and payload.get("audio"):
# Nothing prepared the conversation, so the committed payload is
# the whole turn. An empty prepared list is not that: it means the
# preparation ran and found nothing to say.
history = [{"role": "user", "audio_payload": payload}]
if not history:
raise DuplexRuntimeConfigError("Qwen generation requires committed audio or conversation history")
audios = []
images = []
for index, message in enumerate(history):
if "audio_payload" not in message:
content = message.get("content")
if isinstance(content, list):
parts = []
for part in content:
if part.get("type") == "image_url":
encoded = part["image_url"]["url"].split(",", 1)[1]
with Image.open(io.BytesIO(base64.b64decode(encoded, validate=True))) as image:
image.thumbnail((448, 448))
images.append(image.convert("RGB").copy())
parts.append({"type": "image"})
elif part.get("type") == "text":
parts.append(part)
messages.append({"role": message["role"], "content": parts})
else:
messages.append(message)
continue
content, audio = self._audio_content(message["audio_payload"])
audios.append((audio, 16000))
if runtime_config.get("initial_user_text") and index == len(history) - 1:
content.append({"type": "text", "text": runtime_config["initial_user_text"]})
messages.append({"role": "user", "content": content})
messages, images = self._trim_prompt_images(messages, images)
prompt = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
mm = {"audio": audios} if audios else {}
if images:
mm["image"] = images
logger.info(
"Qwen duplex visual input request=%s images=%d sizes=%s audios=%d",
request_id,
len(images),
[image.size for image in images],
len(audios),
)
# Opt-in capture of the exact decoded images submitted to the processor.
# Request IDs are hashed because they need not be safe path components.
debug_dir = os.environ.get("VLLM_OMNI_QWEN_VISUAL_DEBUG_DIR")
if debug_dir:
try:
target = Path(debug_dir) / hashlib.sha256(request_id.encode()).hexdigest()[:24]
target.mkdir(parents=True, exist_ok=True)
filenames = []
for index, image in enumerate(images):
filename = f"image_{index:02d}.png"
image.save(target / filename)
filenames.append(filename)
(target / "input.json").write_text(
json.dumps(
{"request_id": request_id, "images": filenames, "audio_count": len(audios), "prompt": prompt},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
logger.info("Qwen duplex visual input capture request=%s path=%s", request_id, target)
except (OSError, ValueError):
logger.exception("Failed to capture Qwen visual input request=%s", request_id)
return DuplexAppendPlan(prompt={"prompt": prompt, "multi_modal_data": mm})
def decide_output(self, **kwargs):
return None
def data_plane_context(self, **kwargs):
return kwargs
@@ -0,0 +1,59 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import asyncio
from dataclasses import dataclass, field
from vllm_omni.engine.duplex.plugin import DuplexModelSessionState
from vllm_omni.model_executor.models.qwen3_omni.duplex.input import (
QwenPcmBuffer,
)
@dataclass(slots=True)
class QwenDuplexSessionState(DuplexModelSessionState):
"""Mutable model-owned state of one Qwen duplex session (owned by the session runner)."""
audio_buffer: QwenPcmBuffer = field(default_factory=QwenPcmBuffer)
# Keep content objects alongside payloads so identity survives history copies.
audio_history: list[tuple[object, dict]] = field(default_factory=list)
input_since_commit: bool = False
speech_since_commit: bool = False
context_locked: bool = False
committed_audio_payload: dict[str, object] | None = None
committed_audio_operation_id: str | None = None
committed_audio_reserved_bytes: int = 0
deferred_response_create: bool = False
deferred_precreate_response: bool = False
continuation_owner_id: str | None = None
continuation_units: int = 0
pending_silence_task: asyncio.Task[bool] | None = None
pending_silence_owner_id: str | None = None
def retain_committed_audio(
self,
payload: dict[str, object],
*,
operation_id: str | None,
reserved_bytes: int = 0,
) -> None:
self.committed_audio_payload = payload
self.committed_audio_operation_id = operation_id
self.committed_audio_reserved_bytes += max(0, int(reserved_bytes))
def clear_committed_audio(self) -> int:
reserved_bytes = self.committed_audio_reserved_bytes
self.committed_audio_payload = None
self.committed_audio_operation_id = None
self.committed_audio_reserved_bytes = 0
self.deferred_response_create = False
self.deferred_precreate_response = False
return reserved_bytes
def clear_continuation(self) -> None:
self.continuation_owner_id = None
self.continuation_units = 0
self.pending_silence_task = None
self.pending_silence_owner_id = None
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Qwen3-Omni-MoE pipeline topology (frozen).
Stage 0: Thinker — multimodal understanding + text generation
@@ -21,6 +21,8 @@ _PROC = "vllm_omni.model_executor.stage_input_processors.qwen3_omni"
QWEN3_OMNI_PIPELINE = PipelineConfig(
model_type="qwen3_omni_moe",
default_session_mode="turn",
duplex_plugin="vllm_omni.model_executor.models.qwen3_omni.duplex.plugin.Qwen3OmniDuplexPlugin",
default_deploy_config_name="qwen3_omni_moe.yaml",
model_arch="Qwen3OmniMoeForConditionalGeneration",
endpoint_restrictions=(