[Frontend] Add a shared realtime web UI for MiniCPM-o and Qwen3-Omni (#7222) (#7585)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-09-17 23:32:31 +08:00
1 parent eb1abbadf8
commit 882bf8bc95
53 files changed
+3438 -633

No files matched your search

+34 -18
View File
@@ -521,7 +521,9 @@ def _should_serve_duplex(model: str, kwargs: dict[str, Any]) -> bool:
deploy_path = _DEPLOY_DIR / deploy_path
# Resolve base_config too, so the API and stage workers use the same mode.
session_mode = resolve_deploy_yaml(deploy_path).get("session_mode")
session_mode = resolve_deploy_yaml(deploy_path).get(
"session_mode", getattr(pipeline_config, "default_session_mode", None)
)
if session_mode not in ("turn", "duplex"):
raise ValueError("A duplex-capable model requires session_mode: turn or duplex in its deploy config")
return session_mode == "duplex"
@@ -1742,6 +1744,12 @@ async def realtime_websocket(websocket: WebSocket):
await duplex_handler.handle_realtime_session(websocket)
return
if isinstance(duplex_query, str) and duplex_query.lower() in {"1", "true", "on"}:
await websocket.accept()
await websocket.send_json({"type": "error", "code": "unsupported", "error": "VAD realtime is not enabled"})
await websocket.close(code=1008)
return
serving = getattr(websocket.app.state, "openai_serving_realtime", None)
if serving is None:
await websocket.accept()
@@ -2042,6 +2050,8 @@ async def generate_images(
)
try:
width: int | None = None
height: int | None = None
# Unify request construction for any multi-stage pipeline to avoid
# divergence between /v1/images and /v1/chat/completions.
if len(stage_configs) > 1:
@@ -2059,7 +2069,6 @@ async def generate_images(
"num_outputs_per_prompt": request.n,
}
if request.size is not None:
parse_size(request.size)
width, height = parse_size(request.size)
app_state_args = getattr(raw_request.app.state, "args", None)
_check_max_generated_image_size(app_state_args, width, height)
@@ -2113,7 +2122,7 @@ async def generate_images(
)
# Build params - pass through user values directly
prompt: OmniTextPrompt = {"prompt": request.prompt, "modalities": ["image"]}
prompt = OmniTextPrompt(prompt=request.prompt, modalities=["image"])
if request.negative_prompt is not None:
prompt["negative_prompt"] = request.negative_prompt
gen_params = OmniDiffusionSamplingParams(num_outputs_per_prompt=request.n)
@@ -2129,12 +2138,11 @@ async def generate_images(
if extra_args:
gen_params.extra_args = extra_args
# Parse per-request LoRA (compatible with chat's extra_body.lora shape).
lora_request, lora_scale = _parse_lora_request(request.lora)
lora_request, lora_scale = _parse_lora_request(request.lora) if request.lora is not None else (None, None)
_update_if_not_none(gen_params, "lora_request", lora_request)
_update_if_not_none(gen_params, "lora_scale", lora_scale)
# Parse and add size if provided
width, height = None, None
if request.size:
width, height = parse_size(request.size)
size_str = f"{width}x{height}"
@@ -2300,9 +2308,9 @@ async def edit_images(
try:
# 2. Build prompt & images params
cot_output = None
prompt: OmniTextPrompt = {"prompt": prompt, "modalities": ["image"]}
omni_prompt = OmniTextPrompt(prompt=prompt, modalities=["image"])
if negative_prompt is not None:
prompt["negative_prompt"] = negative_prompt
omni_prompt["negative_prompt"] = negative_prompt
input_images_list = []
images = image or image_array
urls = url or url_array
@@ -2333,17 +2341,17 @@ async def edit_images(
# Hunyuan-aware behavior. RGBA/P uploads otherwise diverge from offline.
normalize_edit_images_rgb = bot_task is not None or sys_type is not None
pil_images = await _load_input_images(input_images_list, normalize_rgb=normalize_edit_images_rgb)
prompt["multi_modal_data"] = {}
prompt["multi_modal_data"]["image"] = pil_images
omni_prompt["multi_modal_data"] = {}
omni_prompt["multi_modal_data"]["image"] = pil_images
if mask_image is not None:
# Mask role is different (alpha channel matters); never normalize.
loaded = await _load_input_images([mask_image], normalize_rgb=False)
prompt["multi_modal_data"]["mask_image"] = loaded[0]
omni_prompt["multi_modal_data"]["mask_image"] = loaded[0]
if reference_image is not None:
loaded = await _load_input_images([reference_image], normalize_rgb=normalize_edit_images_rgb)
prompt["multi_modal_data"]["reference_image"] = loaded[0]
omni_prompt["multi_modal_data"]["reference_image"] = loaded[0]
# 3 Build sample params
gen_params = OmniDiffusionSamplingParams()
@@ -2409,13 +2417,13 @@ async def edit_images(
# Keep AR stage target grid in sync with requested output size.
# GLM-Image consumes target_h/target_w via mm_processor_kwargs.
if width is not None and height is not None:
prompt["mm_processor_kwargs"] = {
omni_prompt["mm_processor_kwargs"] = {
"target_h": height,
"target_w": width,
}
# Backward-compatible fallback for processors reading top-level fields.
prompt["height"] = height
prompt["width"] = width
omni_prompt["height"] = height
omni_prompt["width"] = width
_update_if_not_none(gen_params, "width", width)
_update_if_not_none(gen_params, "height", height)
@@ -2513,7 +2521,7 @@ async def edit_images(
if return_stage_metrics is not None:
extra_body["return_stage_metrics"] = return_stage_metrics
prompt_text = prompt.get("prompt", "")
prompt_text = omni_prompt.get("prompt", "")
generation_result = await chat_handler.generate_diffusion_images(
prompt=prompt_text,
extra_body=extra_body,
@@ -2544,7 +2552,7 @@ async def edit_images(
engine_client=engine_client,
gen_params=gen_params,
stage_configs=stage_configs,
prompt=prompt,
prompt=omni_prompt,
request_id=request_id,
)
images = _extract_images_from_result(result)
@@ -2552,6 +2560,8 @@ async def edit_images(
peak_memory_mb = getattr(result, "peak_memory_mb", None)
response_metrics = getattr(result, "metrics", None) if return_stage_metrics else None
if images is None:
images = []
logger.debug(f"Successfully generated {len(images)} image(s)")
# Encode images to base64
@@ -2973,7 +2983,10 @@ async def omni_sleep(request: OmniSleepRequest, raw_request: Request):
acks = await engine_client.sleep(stage_ids=request.stage_ids, level=request.level)
for sid in request.stage_ids:
sleeping_set.add(sid)
return {"status": "SUCCESS", "acks": [dataclasses.asdict(a) if dataclasses.is_dataclass(a) else a for a in acks]}
return {
"status": "SUCCESS",
"acks": [dataclasses.asdict(a) if dataclasses.is_dataclass(a) and not isinstance(a, type) else a for a in acks],
}
@router.post("/v1/omni/wakeup")
@@ -2989,7 +3002,10 @@ async def omni_wakeup(request: OmniWakeupRequest, raw_request: Request):
for sid in request.stage_ids:
if sid in sleeping_set:
sleeping_set.remove(sid)
return {"status": "SUCCESS", "acks": [dataclasses.asdict(a) if dataclasses.is_dataclass(a) else a for a in acks]}
return {
"status": "SUCCESS",
"acks": [dataclasses.asdict(a) if dataclasses.is_dataclass(a) and not isinstance(a, type) else a for a in acks],
}
if __name__ == "__main__":