[Entrypoint][Refactor] vLLM-Omni Entrypoint Refactoring (#1908)

Signed-off-by: yinpe <11810305@mail.sustech.edu.cn>
Signed-off-by: Peiqi Yin <yinpeiqi809@gmail.com>
Signed-off-by: yinpeiqi <yinpeiqi809@gmail.com>
Signed-off-by: wuhang <whlbx@hotmail.com>
Signed-off-by: linyueqian <linyueqian@outlook.com>
Signed-off-by: princepride <wangzhipeng628@gmail.com>
Signed-off-by: lishunyang <lishunyang12@163.com>
Signed-off-by: Chenguang ZHENG <645327136@qq.com>
Signed-off-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com>
Co-authored-by: yinpe <11810305@mail.sustech.edu.cn>
Co-authored-by: Peiqi Yin <yinpeiqi809@gmail.com>
Co-authored-by: wuhang <whlbx@hotmail.com>
Co-authored-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com>
Co-authored-by: linyueqian <linyueqian@outlook.com>
Co-authored-by: princepride <wangzhipeng628@gmail.com>
Co-authored-by: lishunyang <lishunyang12@163.com>
This commit is contained in:
authored and GitHub committed 2026-03-18 17:33:53 +08:00
1 parent 6a66a1fe36
commit 3158e9d28f
125 files changed
+5210 -9322

No files matched your search

+104 -113
View File
@@ -111,10 +111,9 @@ from vllm_omni.entrypoints.openai.serving_speech_stream import OmniStreamingSpee
from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo, ReferenceImage
from vllm_omni.entrypoints.openai.storage import STORAGE_MANAGER
from vllm_omni.entrypoints.openai.stores import VIDEO_STORE, VIDEO_TASKS
from vllm_omni.entrypoints.openai.utils import get_stage_type, parse_lora_request
from vllm_omni.entrypoints.openai.video_api_utils import decode_input_reference
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniSamplingParams, OmniTextPrompt
from vllm_omni.lora.request import LoRARequest
from vllm_omni.lora.utils import stable_lora_int_id
logger = init_logger(__name__)
router = APIRouter()
@@ -163,6 +162,12 @@ def _remove_route_from_router(
ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL = "endpoint-load-metrics-format"
async def _get_vllm_config(engine_client: EngineClient) -> Any:
if hasattr(engine_client, "get_vllm_config"):
return await engine_client.get_vllm_config()
return getattr(engine_client, "vllm_config", None)
def _remove_route_from_app(app, path: str, methods: set[str] | None = None):
"""Remove a route from the app by path and optionally by methods.
@@ -187,13 +192,31 @@ class _DiffusionServingModels:
provide a lightweight fallback.
"""
class _NullModelConfig:
def __getattr__(self, name):
return None
class _Unsupported:
def __init__(self, name: str):
self.name = name
def __call__(self, *args, **kwargs):
raise NotImplementedError(f"{self.name} is not supported in diffusion mode")
def __getattr__(self, attr):
raise NotImplementedError(f"{self.name}.{attr} is not supported in diffusion mode")
def __init__(self, base_model_paths: list[BaseModelPath]) -> None:
self._base_model_paths = base_model_paths
self.model_config = self._NullModelConfig()
@property
def base_model_paths(self) -> list[BaseModelPath]:
return self._base_model_paths
def __getattr__(self, name):
return self._Unsupported(name)
async def show_available_models(self) -> ModelList:
return ModelList(
data=[
@@ -276,7 +299,7 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
logger.warning("Profiler endpoints are enabled. This should ONLY be used for local development!")
app.include_router(profiler_router)
vllm_config = await engine_client.get_vllm_config()
vllm_config = await _get_vllm_config(engine_client)
# Check if pure diffusion mode (vllm_config will be None)
is_pure_diffusion = vllm_config is None
@@ -334,7 +357,7 @@ async def build_async_omni(
Args:
args: Parsed command-line arguments containing model and configuration
disable_frontend_multiprocessing: Optional flag to disable frontend
multiprocessing (deprecated in V1)
multiprocessing
client_config: Optional client configuration dictionary
Yields:
@@ -372,7 +395,7 @@ async def build_async_omni_from_stage_config(
Args:
args: Parsed command-line arguments containing model and stage configs
disable_frontend_multiprocessing: Flag to disable frontend multiprocessing
(deprecated in V1)
for compatibility with existing CLI options
client_config: Optional client configuration dictionary
Yields:
@@ -383,16 +406,13 @@ async def build_async_omni_from_stage_config(
otherwise from the model's default configuration.
"""
# V1 AsyncLLM.
if disable_frontend_multiprocessing:
logger.warning("V1 is enabled, but got --disable-frontend-multiprocessing.")
logger.warning("Ignoring --disable-frontend-multiprocessing for AsyncOmni runtime.")
async_omni: EngineClient | None = None
try:
# Convert args Namespace to kwargs dict for AsyncOmni to use
kwargs = vars(args).copy()
# Remove model as it will be passed separately
kwargs.pop("model", None)
async_omni = AsyncOmni(model=args.model, **kwargs)
@@ -423,14 +443,14 @@ async def omni_init_app_state(
args: Parsed command-line arguments
"""
# Get vllm_config from engine_client (following 0.14.0 pattern)
vllm_config = await engine_client.get_vllm_config()
vllm_config = await _get_vllm_config(engine_client)
# Detect if it's pure Diffusion mode (single stage and is Diffusion)
is_pure_diffusion = False
if hasattr(engine_client, "stage_configs") and engine_client.stage_configs:
stage_configs = engine_client.stage_configs
if len(stage_configs) == 1:
stage_type = stage_configs[0].get("stage_type", "llm")
stage_type = get_stage_type(stage_configs[0])
if stage_type == "diffusion":
is_pure_diffusion = True
logger.info("Detected pure diffusion mode (single diffusion stage)")
@@ -483,7 +503,7 @@ async def omni_init_app_state(
# LLM or multi-stage mode: use standard initialization logic
if vllm_config is None:
# Try to get vllm_config from engine_client
vllm_config = await engine_client.get_vllm_config()
vllm_config = await _get_vllm_config(engine_client)
if vllm_config is None:
logger.warning("vllm_config is None, some features may not work correctly")
@@ -528,15 +548,13 @@ async def omni_init_app_state(
# Try to initialize processors if vllm_config is available
try:
from vllm.plugins.io_processors import get_io_processor
from vllm_omni.engine.input_processor import OmniInputProcessor
from vllm.v1.engine.input_processor import InputProcessor
tokenizer = await engine_client.get_tokenizer()
if tokenizer is not None:
# Initialize input_processor
# OMNI: OmniInputProcessor creates tokenizer internally from vllm_config
if not hasattr(engine_client, "input_processor") or engine_client.input_processor is None:
engine_client.input_processor = OmniInputProcessor(
engine_client.input_processor = InputProcessor(
vllm_config=vllm_config,
)
logger.info("Initialized input_processor for AsyncOmni")
@@ -1167,7 +1185,7 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
HTTPException: For validation errors, missing engine, or generation failures
"""
# Get engine client (AsyncOmni) from app state
engine_client, model_name, stage_types = _get_engine_and_model(raw_request)
engine_client, model_name, stage_configs = _get_engine_and_model(raw_request)
# Validate model field (warn if mismatch, don't error)
if request.model is not None and request.model != model_name:
@@ -1219,7 +1237,7 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
result = await _generate_with_async_omni(
engine_client=engine_client,
gen_params=gen_params,
stage_types=stage_types,
stage_configs=stage_configs,
prompt=prompt,
request_id=request_id,
)
@@ -1294,7 +1312,7 @@ async def edit_images(
OpenAI-compatible image edit endpoint.
"""
# 1. get engine and model
engine_client, model_name, stage_types = _get_engine_and_model(raw_request)
engine_client, model_name, stage_configs = _get_engine_and_model(raw_request)
if model is not None and model != model_name:
logger.warning(
f"Model mismatch: request specifies '{model}' but server is running '{model_name}'. Using server model."
@@ -1329,8 +1347,14 @@ async def edit_images(
# 3.0 Init with system default values
app_state_args = getattr(raw_request.app.state, "args", None)
default_sample_param = getattr(app_state_args, "default_sampling_params", None)
# Currently only have one diffusion stage
diffusion_stage_id = [i for i, t in enumerate(stage_types) if t == "diffusion"][0]
# Currently only have one diffusion stage.
diffusion_stage_ids = [i for i, cfg in enumerate(stage_configs) if get_stage_type(cfg) == "diffusion"]
if not diffusion_stage_ids:
raise HTTPException(
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
detail="No diffusion stage found in multi-stage pipeline.",
)
diffusion_stage_id = diffusion_stage_ids[0]
apply_stage_default_sampling_params(
default_sample_param,
gen_params,
@@ -1377,7 +1401,7 @@ async def edit_images(
result = await _generate_with_async_omni(
engine_client=engine_client,
gen_params=gen_params,
stage_types=stage_types,
stage_configs=stage_configs,
prompt=prompt,
request_id=request_id,
)
@@ -1418,42 +1442,26 @@ async def edit_images(
def _get_engine_and_model(raw_request: Request):
# Get engine client (AsyncOmni) from app state
engine_client: EngineClient | AsyncOmni | None = getattr(raw_request.app.state, "engine_client", None)
if engine_client is None or not hasattr(engine_client, "stage_list"):
if engine_client is None:
raise HTTPException(
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
detail="Multi-stage engine not initialized. Start server with a multi-stage omni model.",
)
# Check if there's a diffusion stage
# Check if there's a diffusion stage.
# Prefer app state (compat layer populated at startup), then fall back to
# the engine client's stage configs for refactored AsyncOmni paths.
stage_configs = getattr(raw_request.app.state, "stage_configs", None)
if not stage_configs:
stage_configs = getattr(engine_client, "stage_configs", None)
if not stage_configs:
raise HTTPException(
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
detail="Stage configs not found. Start server with a multi-stage omni model.",
)
# Check for diffusion stage and collect stage types
has_diffusion_stage = False
stage_types: list[str] = []
for stage in stage_configs:
# Handle both dict and OmegaConf objects
stage_type = None
if isinstance(stage, dict):
stage_type = stage.get("stage_type", "llm")
elif hasattr(stage, "get"):
stage_type = stage.get("stage_type", "llm")
elif hasattr(stage, "stage_type"):
stage_type = stage.stage_type
else:
# Fallback: try to access as dict-like
try:
stage_type = stage["stage_type"] if "stage_type" in stage else "llm"
except (TypeError, KeyError):
stage_type = "llm"
if stage_type == "diffusion":
has_diffusion_stage = True
stage_types.append(stage_type)
normalized_stage_configs = list(stage_configs)
has_diffusion_stage = any(get_stage_type(stage_cfg) == "diffusion" for stage_cfg in normalized_stage_configs)
if not has_diffusion_stage:
raise HTTPException(
@@ -1463,12 +1471,13 @@ def _get_engine_and_model(raw_request: Request):
# Get server's loaded model name
serving_models = getattr(raw_request.app.state, "openai_serving_models", None)
if serving_models and hasattr(serving_models, "base_model_paths") and serving_models.base_model_paths:
model_name = serving_models.base_model_paths[0].name
base_model_paths = getattr(serving_models, "base_model_paths", None) if serving_models else None
if base_model_paths:
model_name = base_model_paths[0].name
else:
model_name = "unknown"
return engine_client, model_name, stage_types
return engine_client, model_name, normalized_stage_configs
def _get_lora_from_json_str(lora_body):
@@ -1486,81 +1495,63 @@ def _get_lora_from_json_str(lora_body):
def _parse_lora_request(lora_body: dict[str, Any]):
if lora_body is not None:
if not isinstance(lora_body, dict):
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail="Invalid lora field: expected an object.",
)
lora_name = lora_body.get("name") or lora_body.get("lora_name") or lora_body.get("adapter")
lora_path = (
lora_body.get("local_path")
or lora_body.get("path")
or lora_body.get("lora_path")
or lora_body.get("lora_local_path")
)
lora_scale = lora_body.get("scale")
if lora_scale is None:
lora_scale = lora_body.get("lora_scale")
lora_int_id = lora_body.get("int_id")
if lora_int_id is None:
lora_int_id = lora_body.get("lora_int_id")
if lora_int_id is None and lora_path:
lora_int_id = stable_lora_int_id(str(lora_path))
if not lora_name or not lora_path:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail="Invalid lora object: both name and path are required.",
)
return LoRARequest(str(lora_name), int(lora_int_id), str(lora_path)), lora_scale
return None, None
try:
return parse_lora_request(lora_body)
except ValueError as e:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=str(e),
) from e
async def _generate_with_async_omni(
engine_client: AsyncOmni | Any,
gen_params: Any,
stage_types: list[str],
stage_configs: list[Any],
**kwargs,
):
engine_client = cast(AsyncOmni, engine_client)
result = None
stage_list = getattr(engine_client, "stage_list", None)
if isinstance(stage_list, list):
default_params_list: list[OmniSamplingParams] | None = getattr(
engine_client, "default_sampling_params_list", None
normalized_stage_configs = list(stage_configs)
if not normalized_stage_configs:
raise HTTPException(
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
detail="Stage configs not found. Start server with a multi-stage omni model.",
)
if not isinstance(default_params_list, list):
default_params_list = [
OmniDiffusionSamplingParams() if st == "diffusion" else SamplingParams() for st in stage_types
]
else:
default_params_list = list(default_params_list)
if len(default_params_list) != len(stage_types):
default_params_list = (
default_params_list
+ [OmniDiffusionSamplingParams() if st == "diffusion" else SamplingParams() for st in stage_types]
)[: len(stage_types)]
sampling_params_list: list[OmniSamplingParams] = []
for idx, stage_type in enumerate(stage_types):
if stage_type == "diffusion":
sampling_params_list.append(gen_params)
else:
base_params = default_params_list[idx]
sampling_params_list.append(base_params)
async for output in engine_client.generate(
sampling_params_list=sampling_params_list,
**kwargs,
):
result = output
default_params_list: list[OmniSamplingParams] | None = getattr(
engine_client,
"default_sampling_params_list",
None,
)
if not isinstance(default_params_list, list):
default_params_list = []
else:
result = await engine_client.generate(
sampling_params_list=[gen_params],
**kwargs,
)
default_params_list = list(default_params_list)
sampling_params_list: list[OmniSamplingParams] = []
for idx, stage_cfg in enumerate(normalized_stage_configs):
stage_type = get_stage_type(stage_cfg)
if stage_type == "diffusion":
sampling_params_list.append(gen_params)
continue
if idx < len(default_params_list):
default_stage_params = default_params_list[idx]
else:
default_stage_params = SamplingParams()
if hasattr(default_stage_params, "clone"):
try:
default_stage_params = default_stage_params.clone()
except Exception:
pass
sampling_params_list.append(default_stage_params)
async for output in engine_client.generate(
sampling_params_list=sampling_params_list,
**kwargs,
):
result = output
if result is None:
raise HTTPException(