Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
eb1abbadf8
commit
882bf8bc95
53 files changed
+3438
-633
No files matched your search
@@ -521,7 +521,9 @@ def _should_serve_duplex(model: str, kwargs: dict[str, Any]) -> bool:
|
||||
deploy_path = _DEPLOY_DIR / deploy_path
|
||||
|
||||
# Resolve base_config too, so the API and stage workers use the same mode.
|
||||
session_mode = resolve_deploy_yaml(deploy_path).get("session_mode")
|
||||
session_mode = resolve_deploy_yaml(deploy_path).get(
|
||||
"session_mode", getattr(pipeline_config, "default_session_mode", None)
|
||||
)
|
||||
if session_mode not in ("turn", "duplex"):
|
||||
raise ValueError("A duplex-capable model requires session_mode: turn or duplex in its deploy config")
|
||||
return session_mode == "duplex"
|
||||
@@ -1742,6 +1744,12 @@ async def realtime_websocket(websocket: WebSocket):
|
||||
await duplex_handler.handle_realtime_session(websocket)
|
||||
return
|
||||
|
||||
if isinstance(duplex_query, str) and duplex_query.lower() in {"1", "true", "on"}:
|
||||
await websocket.accept()
|
||||
await websocket.send_json({"type": "error", "code": "unsupported", "error": "VAD realtime is not enabled"})
|
||||
await websocket.close(code=1008)
|
||||
return
|
||||
|
||||
serving = getattr(websocket.app.state, "openai_serving_realtime", None)
|
||||
if serving is None:
|
||||
await websocket.accept()
|
||||
@@ -2042,6 +2050,8 @@ async def generate_images(
|
||||
)
|
||||
|
||||
try:
|
||||
width: int | None = None
|
||||
height: int | None = None
|
||||
# Unify request construction for any multi-stage pipeline to avoid
|
||||
# divergence between /v1/images and /v1/chat/completions.
|
||||
if len(stage_configs) > 1:
|
||||
@@ -2059,7 +2069,6 @@ async def generate_images(
|
||||
"num_outputs_per_prompt": request.n,
|
||||
}
|
||||
if request.size is not None:
|
||||
parse_size(request.size)
|
||||
width, height = parse_size(request.size)
|
||||
app_state_args = getattr(raw_request.app.state, "args", None)
|
||||
_check_max_generated_image_size(app_state_args, width, height)
|
||||
@@ -2113,7 +2122,7 @@ async def generate_images(
|
||||
)
|
||||
|
||||
# Build params - pass through user values directly
|
||||
prompt: OmniTextPrompt = {"prompt": request.prompt, "modalities": ["image"]}
|
||||
prompt = OmniTextPrompt(prompt=request.prompt, modalities=["image"])
|
||||
if request.negative_prompt is not None:
|
||||
prompt["negative_prompt"] = request.negative_prompt
|
||||
gen_params = OmniDiffusionSamplingParams(num_outputs_per_prompt=request.n)
|
||||
@@ -2129,12 +2138,11 @@ async def generate_images(
|
||||
if extra_args:
|
||||
gen_params.extra_args = extra_args
|
||||
# Parse per-request LoRA (compatible with chat's extra_body.lora shape).
|
||||
lora_request, lora_scale = _parse_lora_request(request.lora)
|
||||
lora_request, lora_scale = _parse_lora_request(request.lora) if request.lora is not None else (None, None)
|
||||
_update_if_not_none(gen_params, "lora_request", lora_request)
|
||||
_update_if_not_none(gen_params, "lora_scale", lora_scale)
|
||||
|
||||
# Parse and add size if provided
|
||||
width, height = None, None
|
||||
if request.size:
|
||||
width, height = parse_size(request.size)
|
||||
size_str = f"{width}x{height}"
|
||||
@@ -2300,9 +2308,9 @@ async def edit_images(
|
||||
try:
|
||||
# 2. Build prompt & images params
|
||||
cot_output = None
|
||||
prompt: OmniTextPrompt = {"prompt": prompt, "modalities": ["image"]}
|
||||
omni_prompt = OmniTextPrompt(prompt=prompt, modalities=["image"])
|
||||
if negative_prompt is not None:
|
||||
prompt["negative_prompt"] = negative_prompt
|
||||
omni_prompt["negative_prompt"] = negative_prompt
|
||||
input_images_list = []
|
||||
images = image or image_array
|
||||
urls = url or url_array
|
||||
@@ -2333,17 +2341,17 @@ async def edit_images(
|
||||
# Hunyuan-aware behavior. RGBA/P uploads otherwise diverge from offline.
|
||||
normalize_edit_images_rgb = bot_task is not None or sys_type is not None
|
||||
pil_images = await _load_input_images(input_images_list, normalize_rgb=normalize_edit_images_rgb)
|
||||
prompt["multi_modal_data"] = {}
|
||||
prompt["multi_modal_data"]["image"] = pil_images
|
||||
omni_prompt["multi_modal_data"] = {}
|
||||
omni_prompt["multi_modal_data"]["image"] = pil_images
|
||||
|
||||
if mask_image is not None:
|
||||
# Mask role is different (alpha channel matters); never normalize.
|
||||
loaded = await _load_input_images([mask_image], normalize_rgb=False)
|
||||
prompt["multi_modal_data"]["mask_image"] = loaded[0]
|
||||
omni_prompt["multi_modal_data"]["mask_image"] = loaded[0]
|
||||
|
||||
if reference_image is not None:
|
||||
loaded = await _load_input_images([reference_image], normalize_rgb=normalize_edit_images_rgb)
|
||||
prompt["multi_modal_data"]["reference_image"] = loaded[0]
|
||||
omni_prompt["multi_modal_data"]["reference_image"] = loaded[0]
|
||||
|
||||
# 3 Build sample params
|
||||
gen_params = OmniDiffusionSamplingParams()
|
||||
@@ -2409,13 +2417,13 @@ async def edit_images(
|
||||
# Keep AR stage target grid in sync with requested output size.
|
||||
# GLM-Image consumes target_h/target_w via mm_processor_kwargs.
|
||||
if width is not None and height is not None:
|
||||
prompt["mm_processor_kwargs"] = {
|
||||
omni_prompt["mm_processor_kwargs"] = {
|
||||
"target_h": height,
|
||||
"target_w": width,
|
||||
}
|
||||
# Backward-compatible fallback for processors reading top-level fields.
|
||||
prompt["height"] = height
|
||||
prompt["width"] = width
|
||||
omni_prompt["height"] = height
|
||||
omni_prompt["width"] = width
|
||||
|
||||
_update_if_not_none(gen_params, "width", width)
|
||||
_update_if_not_none(gen_params, "height", height)
|
||||
@@ -2513,7 +2521,7 @@ async def edit_images(
|
||||
if return_stage_metrics is not None:
|
||||
extra_body["return_stage_metrics"] = return_stage_metrics
|
||||
|
||||
prompt_text = prompt.get("prompt", "")
|
||||
prompt_text = omni_prompt.get("prompt", "")
|
||||
generation_result = await chat_handler.generate_diffusion_images(
|
||||
prompt=prompt_text,
|
||||
extra_body=extra_body,
|
||||
@@ -2544,7 +2552,7 @@ async def edit_images(
|
||||
engine_client=engine_client,
|
||||
gen_params=gen_params,
|
||||
stage_configs=stage_configs,
|
||||
prompt=prompt,
|
||||
prompt=omni_prompt,
|
||||
request_id=request_id,
|
||||
)
|
||||
images = _extract_images_from_result(result)
|
||||
@@ -2552,6 +2560,8 @@ async def edit_images(
|
||||
peak_memory_mb = getattr(result, "peak_memory_mb", None)
|
||||
response_metrics = getattr(result, "metrics", None) if return_stage_metrics else None
|
||||
|
||||
if images is None:
|
||||
images = []
|
||||
logger.debug(f"Successfully generated {len(images)} image(s)")
|
||||
|
||||
# Encode images to base64
|
||||
@@ -2973,7 +2983,10 @@ async def omni_sleep(request: OmniSleepRequest, raw_request: Request):
|
||||
acks = await engine_client.sleep(stage_ids=request.stage_ids, level=request.level)
|
||||
for sid in request.stage_ids:
|
||||
sleeping_set.add(sid)
|
||||
return {"status": "SUCCESS", "acks": [dataclasses.asdict(a) if dataclasses.is_dataclass(a) else a for a in acks]}
|
||||
return {
|
||||
"status": "SUCCESS",
|
||||
"acks": [dataclasses.asdict(a) if dataclasses.is_dataclass(a) and not isinstance(a, type) else a for a in acks],
|
||||
}
|
||||
|
||||
|
||||
@router.post("/v1/omni/wakeup")
|
||||
@@ -2989,7 +3002,10 @@ async def omni_wakeup(request: OmniWakeupRequest, raw_request: Request):
|
||||
for sid in request.stage_ids:
|
||||
if sid in sleeping_set:
|
||||
sleeping_set.remove(sid)
|
||||
return {"status": "SUCCESS", "acks": [dataclasses.asdict(a) if dataclasses.is_dataclass(a) else a for a in acks]}
|
||||
return {
|
||||
"status": "SUCCESS",
|
||||
"acks": [dataclasses.asdict(a) if dataclasses.is_dataclass(a) and not isinstance(a, type) else a for a in acks],
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in new issue
Block a user