[Enhancement] Engine runtime errors (#2426)

Signed-off-by: Daniel Huang <daniel1.huang@intel.com>
Signed-off-by: Daniel Huang <pilotflyer824@gmail.com>
Signed-off-by: bjf-frz <frz123db@gmail.com>
Co-authored-by: Chenguang Zheng <645327136@qq.com>
Co-authored-by: Zhou Taichang <tzhouam@connect.ust.hk>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: bjf-frz <frz123db@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-04-21 14:46:05 +08:00
1 parent c1ba86ab5b
commit 52b53367ef
23 files changed
+1597 -98

No files matched your search

+162 -28
View File
@@ -31,7 +31,7 @@ from vllm import SamplingParams
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.entrypoints.launcher import serve_http
from vllm.entrypoints.launcher import serve_http, terminate_if_errored
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.mcp.tool_server import DemoToolServer, MCPToolServer, ToolServer
from vllm.entrypoints.openai.api_server import build_app as build_openai_app
@@ -50,6 +50,7 @@ from vllm.entrypoints.openai.engine.protocol import (
ModelCard,
ModelList,
ModelPermission,
RequestResponseMetadata,
)
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
@@ -74,6 +75,7 @@ from vllm.entrypoints.serve.instrumentator.basic import base
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
from vllm.entrypoints.utils import (
create_error_response,
load_aware_call,
process_lora_modules,
with_cancellation,
@@ -83,6 +85,7 @@ from vllm.tasks import POOLING_TASKS
from vllm.tool_parsers import ToolParserManager
from vllm.utils import random_uuid
from vllm.utils.system_utils import decorate_logs
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.openai.errors import InvalidInputReferenceError
@@ -199,6 +202,50 @@ def _remove_route_from_app(app, path: str, methods: set[str] | None = None):
app.routes.remove(route)
def _register_omni_exception_handlers(app) -> None:
"""Override upstream vLLM exception handlers with Omni-aware versions.
The upstream ``engine_error_handler`` is designed for ``AsyncLLM`` (single
EngineCore process). Omni uses a multi-stage orchestrator with different
health semantics, so we register our own handlers that:
- Log multi-stage diagnostic info (orchestrator liveness, per-stage health)
when an ``EngineDeadError`` is caught.
- Call ``terminate_if_errored``
- Return an OpenAI-compatible error JSON response.
"""
async def omni_engine_error_handler(
req: Request,
exc: EngineDeadError | EngineGenerateError,
):
request_id = req.state.request_metadata.request_id if hasattr(req.state, "request_metadata") else None
if req.app.state.args.log_error_stack:
logger.exception("Engine Exception caught. Request id: %s", request_id)
engine = req.app.state.engine_client
if isinstance(exc, EngineDeadError):
# Log Omni-specific diagnostic information for dead engines.
orchestrator_alive = engine.engine.is_alive() if hasattr(engine, "engine") else "N/A"
logger.error(
"EngineDeadError: orchestrator_alive=%s, errored=%s, request_id=%s",
orchestrator_alive,
engine.errored,
request_id,
)
terminate_if_errored(
server=req.app.state.server,
engine=engine,
)
err = create_error_response(exc)
return JSONResponse(err.model_dump(), status_code=err.error.code)
app.exception_handler(EngineGenerateError)(omni_engine_error_handler)
app.exception_handler(EngineDeadError)(omni_engine_error_handler)
class _DiffusionServingModels:
"""Minimal OpenAIServingModels implementation for diffusion-only servers.
@@ -308,6 +355,10 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
_remove_route_from_app(app, "/v1/models", {"GET"}) # Remove upstream /v1/models to use omni's handler
app.include_router(router)
# OMNI: Override upstream exception handlers with Omni-aware versions
# that understand the multi-stage orchestrator lifecycle.
_register_omni_exception_handlers(app)
await omni_init_app_state(engine_client, app.state, args)
# Conditionally register profiler endpoints based on stage YAML configs
@@ -878,6 +929,8 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
return base_server.create_error_response(message="The model does not support Chat Completions API")
try:
generator = await handler.create_chat_completion(request, raw_request)
except (EngineGenerateError, EngineDeadError):
raise # Propagate to the global Omni exception handler
except Exception as e:
logger.exception("Chat completion failed: %s", e)
raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e
@@ -973,6 +1026,8 @@ async def create_speech(request: OpenAICreateSpeechRequest, raw_request: Request
status_code=result.error.code if result.error else 400,
)
return result
except (EngineGenerateError, EngineDeadError):
raise # Propagate to the global Omni exception handler
except Exception as e:
raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e
@@ -1007,6 +1062,8 @@ async def create_speech_batch(request: BatchSpeechRequest, raw_request: Request)
status_code=result.error.code if result.error else 400,
)
return JSONResponse(content=result.model_dump())
except (EngineGenerateError, EngineDeadError):
raise # Propagate to the global Omni exception handler
except ValueError as e:
raise HTTPException(status_code=HTTPStatus.BAD_REQUEST.value, detail=str(e)) from e
except Exception as e:
@@ -1244,31 +1301,26 @@ _remove_route_from_router(router, "/health")
async def health(raw_request: Request) -> JSONResponse:
"""Health check endpoint that works for both LLM and diffusion modes.
Returns 200 OK if the server is healthy.
For LLM mode: delegates to engine_client health check
For diffusion mode: checks if diffusion_engine is running
Returns 200 OK if the server is healthy, 503 if the engine is dead.
Mirrors vLLM upstream's /health which catches EngineDeadError -> 503.
"""
# Check if we're in diffusion mode
diffusion_engine = getattr(raw_request.app.state, "diffusion_engine", None)
if diffusion_engine is not None:
# Diffusion mode health check
if hasattr(diffusion_engine, "is_running") and diffusion_engine.is_running:
return JSONResponse(content={"status": "healthy"})
engine_client = getattr(raw_request.app.state, "engine_client", None) or getattr(
raw_request.app.state, "diffusion_engine", None
)
if engine_client is None:
return JSONResponse(
content={"status": "unhealthy", "reason": "Diffusion engine is not running"},
content={"status": "unhealthy", "reason": "No engine initialized"},
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
)
# LLM mode - delegate to engine_client
engine_client = getattr(raw_request.app.state, "engine_client", None)
if engine_client is not None:
try:
await engine_client.check_health()
return JSONResponse(content={"status": "healthy"})
return JSONResponse(
content={"status": "unhealthy", "reason": "No engine initialized"},
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
)
except EngineDeadError:
return JSONResponse(
content={"status": "unhealthy"},
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
)
# Remove existing models endpoint if present (from vllm imports)
@@ -1437,6 +1489,7 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
_update_if_not_none(gen_params, "layers", request.layers)
request_id = f"img_gen-{random_uuid()}"
raw_request.state.request_metadata = RequestResponseMetadata(request_id=request_id)
logger.info(f"Generating {request.n} image(s) {size_str}")
@@ -1468,8 +1521,9 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
data=image_data,
)
except (EngineGenerateError, EngineDeadError):
raise # Propagate to the global Omni exception handler
except HTTPException:
# Re-raise HTTPExceptions as-is
raise
except ValueError as e:
logger.error(f"Validation error: {e}")
@@ -1672,6 +1726,7 @@ async def edit_images(
# 4. Generate images using AsyncOmni (multi-stage mode)
request_id = f"img_edit-{random_uuid()}"
raw_request.state.request_metadata = RequestResponseMetadata(request_id=request_id)
logger.info(f"Generating {n} image(s) {size_str}")
result = await _generate_with_async_omni(
engine_client=engine_client,
@@ -1703,8 +1758,9 @@ async def edit_images(
size=size_str,
)
except (EngineGenerateError, EngineDeadError):
raise # Propagate to the global Omni exception handler
except HTTPException:
# Re-raise HTTPExceptions as-is
raise
except ValueError as e:
logger.error(f"Validation error: {e}")
@@ -2110,6 +2166,48 @@ def video_response_from_request(model_name: str, req: VideoGenerationRequest) ->
return resp
def _status_code_for_video_failure(error: VideoError | None) -> int:
if error is None:
return HTTPStatus.INTERNAL_SERVER_ERROR.value
if isinstance(error.code, int):
if 400 <= error.code < 600:
return error.code
return HTTPStatus.INTERNAL_SERVER_ERROR.value
if error.code == "HTTPException":
status_text, _, _ = error.message.partition(":")
try:
status_code = int(status_text)
except ValueError:
return HTTPStatus.INTERNAL_SERVER_ERROR.value
if 400 <= status_code < 600:
return status_code
return HTTPStatus.INTERNAL_SERVER_ERROR.value
if error.code == "EngineDeadError":
return HTTPStatus.INTERNAL_SERVER_ERROR.value
if error.code == "EngineGenerateError":
return HTTPStatus.INTERNAL_SERVER_ERROR.value
return HTTPStatus.INTERNAL_SERVER_ERROR.value
def _video_error_from_exception(exc: Exception) -> VideoError:
if isinstance(exc, HTTPException):
message = str(exc.detail) if exc.detail else str(exc)
return VideoError(code=exc.status_code, message=message)
if isinstance(exc, (EngineGenerateError, EngineDeadError)):
err = create_error_response(exc)
return VideoError(code=err.error.code, message=err.error.message)
return VideoError(
code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
message=str(exc),
)
def _cleanup_video(video_id: str, output_path: str | None):
try:
if output_path is not None:
@@ -2123,6 +2221,7 @@ async def _run_video_generation_job(
request: VideoGenerationRequest,
video_id: str,
reference_image: ReferenceImage | None = None,
app_state: Any | None = None,
) -> None:
job = await VIDEO_STORE.get(video_id)
if job is None:
@@ -2153,17 +2252,36 @@ async def _run_video_generation_job(
"peak_memory_mb": peak_memory_mb,
},
)
except Exception as exc:
logger.exception("Video generation failed for id=%s", video_id)
except (EngineGenerateError, EngineDeadError) as exc:
logger.exception("Video generation failed (engine error) for id=%s", video_id)
_cleanup_video(video_id, output_path)
# TODO: It would be better to have a finite collection of errors to return rather than the exception name
await VIDEO_STORE.update_fields(
video_id,
{
"status": VideoGenerationStatus.FAILED,
"completed_at": int(time.time()),
"error": VideoError(code=type(exc).__name__, message=str(exc)),
"error": _video_error_from_exception(exc),
"inference_time_s": time.perf_counter() - started_at,
},
)
# Background tasks can't propagate exceptions to FastAPI handlers.
# Actively signal shutdown when the engine is dead.
if app_state is not None and isinstance(exc, EngineDeadError):
terminate_if_errored(
server=app_state.server,
engine=app_state.engine_client,
)
except Exception as exc:
logger.exception("Video generation failed for id=%s", video_id)
_cleanup_video(video_id, output_path)
await VIDEO_STORE.update_fields(
video_id,
{
"status": VideoGenerationStatus.FAILED,
"completed_at": int(time.time()),
"error": _video_error_from_exception(exc),
"inference_time_s": time.perf_counter() - started_at,
},
)
@@ -2294,6 +2412,7 @@ async def _parse_video_form(
},
)
async def create_video(
raw_request: Request,
ctx: tuple[VideoGenerationRequest, OmniOpenAIServingVideo, str, ReferenceImage | None] = Depends(_parse_video_form),
) -> VideoResponse:
"""Create an asynchronous video generation job.
@@ -2304,7 +2423,9 @@ async def create_video(
request, handler, effective_model_name, reference_image = ctx
ref = video_response_from_request(effective_model_name, request)
await VIDEO_STORE.upsert(ref.id, ref)
task = asyncio.create_task(_run_video_generation_job(handler, request, ref.id, reference_image))
task = asyncio.create_task(
_run_video_generation_job(handler, request, ref.id, reference_image, app_state=raw_request.app.state)
)
await VIDEO_TASKS.upsert(ref.id, task)
return ref
@@ -2319,6 +2440,7 @@ async def create_video(
},
)
async def create_video_sync(
raw_request: Request,
ctx: tuple[VideoGenerationRequest, OmniOpenAIServingVideo, str, ReferenceImage | None] = Depends(_parse_video_form),
) -> Response:
"""Synchronous video generation endpoint.
@@ -2332,6 +2454,7 @@ async def create_video_sync(
"""
request, handler, effective_model_name, reference_image = ctx
request_id = f"video_sync-{random_uuid()}"
raw_request.state.request_metadata = RequestResponseMetadata(request_id=request_id)
started_at = time.perf_counter()
try:
video_bytes, stage_durations, peak_memory_mb = await asyncio.wait_for(
@@ -2343,6 +2466,8 @@ async def create_video_sync(
status_code=HTTPStatus.GATEWAY_TIMEOUT.value,
detail=f"Video generation timed out after {VIDEO_SYNC_TIMEOUT_S}s.",
)
except (EngineGenerateError, EngineDeadError):
raise # Propagate to the global Omni exception handler
except HTTPException:
raise
except Exception as exc:
@@ -2403,8 +2528,8 @@ async def list_videos(
return VideoListResponse(data=jobs, has_more=has_more, first_id=first_id, last_id=last_id)
@router.get("/v1/videos/{video_id}")
async def retrieve_video(video_id: str) -> VideoResponse:
@router.get("/v1/videos/{video_id}", response_model=None)
async def retrieve_video(video_id: str) -> VideoResponse | JSONResponse:
"""Retrieve metadata for a previously created video job.
Args:
@@ -2419,6 +2544,15 @@ async def retrieve_video(video_id: str) -> VideoResponse:
job = await VIDEO_STORE.get(video_id)
if job is None:
raise HTTPException(status_code=404, detail="Video not found")
if job.status == VideoGenerationStatus.FAILED:
status_code = _status_code_for_video_failure(job.error)
content = job.model_dump(mode="json")
if content.get("error") is not None:
content["error"]["code"] = status_code
return JSONResponse(
content=content,
status_code=status_code,
)
return job