[Entrypoint][Refactor] vLLM-Omni Entrypoint Refactoring (#1908)
Signed-off-by: yinpe <11810305@mail.sustech.edu.cn> Signed-off-by: Peiqi Yin <yinpeiqi809@gmail.com> Signed-off-by: yinpeiqi <yinpeiqi809@gmail.com> Signed-off-by: wuhang <whlbx@hotmail.com> Signed-off-by: linyueqian <linyueqian@outlook.com> Signed-off-by: princepride <wangzhipeng628@gmail.com> Signed-off-by: lishunyang <lishunyang12@163.com> Signed-off-by: Chenguang ZHENG <645327136@qq.com> Signed-off-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com> Co-authored-by: yinpe <11810305@mail.sustech.edu.cn> Co-authored-by: Peiqi Yin <yinpeiqi809@gmail.com> Co-authored-by: wuhang <whlbx@hotmail.com> Co-authored-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com> Co-authored-by: linyueqian <linyueqian@outlook.com> Co-authored-by: princepride <wangzhipeng628@gmail.com> Co-authored-by: lishunyang <lishunyang12@163.com>
This commit is contained in:
125 files changed
+5210
-9322
No files matched your search
@@ -111,10 +111,9 @@ from vllm_omni.entrypoints.openai.serving_speech_stream import OmniStreamingSpee
|
||||
from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo, ReferenceImage
|
||||
from vllm_omni.entrypoints.openai.storage import STORAGE_MANAGER
|
||||
from vllm_omni.entrypoints.openai.stores import VIDEO_STORE, VIDEO_TASKS
|
||||
from vllm_omni.entrypoints.openai.utils import get_stage_type, parse_lora_request
|
||||
from vllm_omni.entrypoints.openai.video_api_utils import decode_input_reference
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniSamplingParams, OmniTextPrompt
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.lora.utils import stable_lora_int_id
|
||||
|
||||
logger = init_logger(__name__)
|
||||
router = APIRouter()
|
||||
@@ -163,6 +162,12 @@ def _remove_route_from_router(
|
||||
ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL = "endpoint-load-metrics-format"
|
||||
|
||||
|
||||
async def _get_vllm_config(engine_client: EngineClient) -> Any:
|
||||
if hasattr(engine_client, "get_vllm_config"):
|
||||
return await engine_client.get_vllm_config()
|
||||
return getattr(engine_client, "vllm_config", None)
|
||||
|
||||
|
||||
def _remove_route_from_app(app, path: str, methods: set[str] | None = None):
|
||||
"""Remove a route from the app by path and optionally by methods.
|
||||
|
||||
@@ -187,13 +192,31 @@ class _DiffusionServingModels:
|
||||
provide a lightweight fallback.
|
||||
"""
|
||||
|
||||
class _NullModelConfig:
|
||||
def __getattr__(self, name):
|
||||
return None
|
||||
|
||||
class _Unsupported:
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
raise NotImplementedError(f"{self.name} is not supported in diffusion mode")
|
||||
|
||||
def __getattr__(self, attr):
|
||||
raise NotImplementedError(f"{self.name}.{attr} is not supported in diffusion mode")
|
||||
|
||||
def __init__(self, base_model_paths: list[BaseModelPath]) -> None:
|
||||
self._base_model_paths = base_model_paths
|
||||
self.model_config = self._NullModelConfig()
|
||||
|
||||
@property
|
||||
def base_model_paths(self) -> list[BaseModelPath]:
|
||||
return self._base_model_paths
|
||||
|
||||
def __getattr__(self, name):
|
||||
return self._Unsupported(name)
|
||||
|
||||
async def show_available_models(self) -> ModelList:
|
||||
return ModelList(
|
||||
data=[
|
||||
@@ -276,7 +299,7 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
|
||||
logger.warning("Profiler endpoints are enabled. This should ONLY be used for local development!")
|
||||
app.include_router(profiler_router)
|
||||
|
||||
vllm_config = await engine_client.get_vllm_config()
|
||||
vllm_config = await _get_vllm_config(engine_client)
|
||||
|
||||
# Check if pure diffusion mode (vllm_config will be None)
|
||||
is_pure_diffusion = vllm_config is None
|
||||
@@ -334,7 +357,7 @@ async def build_async_omni(
|
||||
Args:
|
||||
args: Parsed command-line arguments containing model and configuration
|
||||
disable_frontend_multiprocessing: Optional flag to disable frontend
|
||||
multiprocessing (deprecated in V1)
|
||||
multiprocessing
|
||||
client_config: Optional client configuration dictionary
|
||||
|
||||
Yields:
|
||||
@@ -372,7 +395,7 @@ async def build_async_omni_from_stage_config(
|
||||
Args:
|
||||
args: Parsed command-line arguments containing model and stage configs
|
||||
disable_frontend_multiprocessing: Flag to disable frontend multiprocessing
|
||||
(deprecated in V1)
|
||||
for compatibility with existing CLI options
|
||||
client_config: Optional client configuration dictionary
|
||||
|
||||
Yields:
|
||||
@@ -383,16 +406,13 @@ async def build_async_omni_from_stage_config(
|
||||
otherwise from the model's default configuration.
|
||||
"""
|
||||
|
||||
# V1 AsyncLLM.
|
||||
if disable_frontend_multiprocessing:
|
||||
logger.warning("V1 is enabled, but got --disable-frontend-multiprocessing.")
|
||||
logger.warning("Ignoring --disable-frontend-multiprocessing for AsyncOmni runtime.")
|
||||
|
||||
async_omni: EngineClient | None = None
|
||||
|
||||
try:
|
||||
# Convert args Namespace to kwargs dict for AsyncOmni to use
|
||||
kwargs = vars(args).copy()
|
||||
# Remove model as it will be passed separately
|
||||
kwargs.pop("model", None)
|
||||
async_omni = AsyncOmni(model=args.model, **kwargs)
|
||||
|
||||
@@ -423,14 +443,14 @@ async def omni_init_app_state(
|
||||
args: Parsed command-line arguments
|
||||
"""
|
||||
# Get vllm_config from engine_client (following 0.14.0 pattern)
|
||||
vllm_config = await engine_client.get_vllm_config()
|
||||
vllm_config = await _get_vllm_config(engine_client)
|
||||
|
||||
# Detect if it's pure Diffusion mode (single stage and is Diffusion)
|
||||
is_pure_diffusion = False
|
||||
if hasattr(engine_client, "stage_configs") and engine_client.stage_configs:
|
||||
stage_configs = engine_client.stage_configs
|
||||
if len(stage_configs) == 1:
|
||||
stage_type = stage_configs[0].get("stage_type", "llm")
|
||||
stage_type = get_stage_type(stage_configs[0])
|
||||
if stage_type == "diffusion":
|
||||
is_pure_diffusion = True
|
||||
logger.info("Detected pure diffusion mode (single diffusion stage)")
|
||||
@@ -483,7 +503,7 @@ async def omni_init_app_state(
|
||||
# LLM or multi-stage mode: use standard initialization logic
|
||||
if vllm_config is None:
|
||||
# Try to get vllm_config from engine_client
|
||||
vllm_config = await engine_client.get_vllm_config()
|
||||
vllm_config = await _get_vllm_config(engine_client)
|
||||
if vllm_config is None:
|
||||
logger.warning("vllm_config is None, some features may not work correctly")
|
||||
|
||||
@@ -528,15 +548,13 @@ async def omni_init_app_state(
|
||||
# Try to initialize processors if vllm_config is available
|
||||
try:
|
||||
from vllm.plugins.io_processors import get_io_processor
|
||||
|
||||
from vllm_omni.engine.input_processor import OmniInputProcessor
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
|
||||
tokenizer = await engine_client.get_tokenizer()
|
||||
if tokenizer is not None:
|
||||
# Initialize input_processor
|
||||
# OMNI: OmniInputProcessor creates tokenizer internally from vllm_config
|
||||
if not hasattr(engine_client, "input_processor") or engine_client.input_processor is None:
|
||||
engine_client.input_processor = OmniInputProcessor(
|
||||
engine_client.input_processor = InputProcessor(
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
logger.info("Initialized input_processor for AsyncOmni")
|
||||
@@ -1167,7 +1185,7 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
|
||||
HTTPException: For validation errors, missing engine, or generation failures
|
||||
"""
|
||||
# Get engine client (AsyncOmni) from app state
|
||||
engine_client, model_name, stage_types = _get_engine_and_model(raw_request)
|
||||
engine_client, model_name, stage_configs = _get_engine_and_model(raw_request)
|
||||
|
||||
# Validate model field (warn if mismatch, don't error)
|
||||
if request.model is not None and request.model != model_name:
|
||||
@@ -1219,7 +1237,7 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request)
|
||||
result = await _generate_with_async_omni(
|
||||
engine_client=engine_client,
|
||||
gen_params=gen_params,
|
||||
stage_types=stage_types,
|
||||
stage_configs=stage_configs,
|
||||
prompt=prompt,
|
||||
request_id=request_id,
|
||||
)
|
||||
@@ -1294,7 +1312,7 @@ async def edit_images(
|
||||
OpenAI-compatible image edit endpoint.
|
||||
"""
|
||||
# 1. get engine and model
|
||||
engine_client, model_name, stage_types = _get_engine_and_model(raw_request)
|
||||
engine_client, model_name, stage_configs = _get_engine_and_model(raw_request)
|
||||
if model is not None and model != model_name:
|
||||
logger.warning(
|
||||
f"Model mismatch: request specifies '{model}' but server is running '{model_name}'. Using server model."
|
||||
@@ -1329,8 +1347,14 @@ async def edit_images(
|
||||
# 3.0 Init with system default values
|
||||
app_state_args = getattr(raw_request.app.state, "args", None)
|
||||
default_sample_param = getattr(app_state_args, "default_sampling_params", None)
|
||||
# Currently only have one diffusion stage
|
||||
diffusion_stage_id = [i for i, t in enumerate(stage_types) if t == "diffusion"][0]
|
||||
# Currently only have one diffusion stage.
|
||||
diffusion_stage_ids = [i for i, cfg in enumerate(stage_configs) if get_stage_type(cfg) == "diffusion"]
|
||||
if not diffusion_stage_ids:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
|
||||
detail="No diffusion stage found in multi-stage pipeline.",
|
||||
)
|
||||
diffusion_stage_id = diffusion_stage_ids[0]
|
||||
apply_stage_default_sampling_params(
|
||||
default_sample_param,
|
||||
gen_params,
|
||||
@@ -1377,7 +1401,7 @@ async def edit_images(
|
||||
result = await _generate_with_async_omni(
|
||||
engine_client=engine_client,
|
||||
gen_params=gen_params,
|
||||
stage_types=stage_types,
|
||||
stage_configs=stage_configs,
|
||||
prompt=prompt,
|
||||
request_id=request_id,
|
||||
)
|
||||
@@ -1418,42 +1442,26 @@ async def edit_images(
|
||||
def _get_engine_and_model(raw_request: Request):
|
||||
# Get engine client (AsyncOmni) from app state
|
||||
engine_client: EngineClient | AsyncOmni | None = getattr(raw_request.app.state, "engine_client", None)
|
||||
if engine_client is None or not hasattr(engine_client, "stage_list"):
|
||||
if engine_client is None:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
|
||||
detail="Multi-stage engine not initialized. Start server with a multi-stage omni model.",
|
||||
)
|
||||
|
||||
# Check if there's a diffusion stage
|
||||
# Check if there's a diffusion stage.
|
||||
# Prefer app state (compat layer populated at startup), then fall back to
|
||||
# the engine client's stage configs for refactored AsyncOmni paths.
|
||||
stage_configs = getattr(raw_request.app.state, "stage_configs", None)
|
||||
if not stage_configs:
|
||||
stage_configs = getattr(engine_client, "stage_configs", None)
|
||||
if not stage_configs:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
|
||||
detail="Stage configs not found. Start server with a multi-stage omni model.",
|
||||
)
|
||||
|
||||
# Check for diffusion stage and collect stage types
|
||||
has_diffusion_stage = False
|
||||
stage_types: list[str] = []
|
||||
for stage in stage_configs:
|
||||
# Handle both dict and OmegaConf objects
|
||||
stage_type = None
|
||||
if isinstance(stage, dict):
|
||||
stage_type = stage.get("stage_type", "llm")
|
||||
elif hasattr(stage, "get"):
|
||||
stage_type = stage.get("stage_type", "llm")
|
||||
elif hasattr(stage, "stage_type"):
|
||||
stage_type = stage.stage_type
|
||||
else:
|
||||
# Fallback: try to access as dict-like
|
||||
try:
|
||||
stage_type = stage["stage_type"] if "stage_type" in stage else "llm"
|
||||
except (TypeError, KeyError):
|
||||
stage_type = "llm"
|
||||
|
||||
if stage_type == "diffusion":
|
||||
has_diffusion_stage = True
|
||||
stage_types.append(stage_type)
|
||||
normalized_stage_configs = list(stage_configs)
|
||||
has_diffusion_stage = any(get_stage_type(stage_cfg) == "diffusion" for stage_cfg in normalized_stage_configs)
|
||||
|
||||
if not has_diffusion_stage:
|
||||
raise HTTPException(
|
||||
@@ -1463,12 +1471,13 @@ def _get_engine_and_model(raw_request: Request):
|
||||
|
||||
# Get server's loaded model name
|
||||
serving_models = getattr(raw_request.app.state, "openai_serving_models", None)
|
||||
if serving_models and hasattr(serving_models, "base_model_paths") and serving_models.base_model_paths:
|
||||
model_name = serving_models.base_model_paths[0].name
|
||||
base_model_paths = getattr(serving_models, "base_model_paths", None) if serving_models else None
|
||||
if base_model_paths:
|
||||
model_name = base_model_paths[0].name
|
||||
else:
|
||||
model_name = "unknown"
|
||||
|
||||
return engine_client, model_name, stage_types
|
||||
return engine_client, model_name, normalized_stage_configs
|
||||
|
||||
|
||||
def _get_lora_from_json_str(lora_body):
|
||||
@@ -1486,81 +1495,63 @@ def _get_lora_from_json_str(lora_body):
|
||||
|
||||
|
||||
def _parse_lora_request(lora_body: dict[str, Any]):
|
||||
if lora_body is not None:
|
||||
if not isinstance(lora_body, dict):
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail="Invalid lora field: expected an object.",
|
||||
)
|
||||
lora_name = lora_body.get("name") or lora_body.get("lora_name") or lora_body.get("adapter")
|
||||
lora_path = (
|
||||
lora_body.get("local_path")
|
||||
or lora_body.get("path")
|
||||
or lora_body.get("lora_path")
|
||||
or lora_body.get("lora_local_path")
|
||||
)
|
||||
lora_scale = lora_body.get("scale")
|
||||
if lora_scale is None:
|
||||
lora_scale = lora_body.get("lora_scale")
|
||||
lora_int_id = lora_body.get("int_id")
|
||||
if lora_int_id is None:
|
||||
lora_int_id = lora_body.get("lora_int_id")
|
||||
if lora_int_id is None and lora_path:
|
||||
lora_int_id = stable_lora_int_id(str(lora_path))
|
||||
|
||||
if not lora_name or not lora_path:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail="Invalid lora object: both name and path are required.",
|
||||
)
|
||||
|
||||
return LoRARequest(str(lora_name), int(lora_int_id), str(lora_path)), lora_scale
|
||||
return None, None
|
||||
try:
|
||||
return parse_lora_request(lora_body)
|
||||
except ValueError as e:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail=str(e),
|
||||
) from e
|
||||
|
||||
|
||||
async def _generate_with_async_omni(
|
||||
engine_client: AsyncOmni | Any,
|
||||
gen_params: Any,
|
||||
stage_types: list[str],
|
||||
stage_configs: list[Any],
|
||||
**kwargs,
|
||||
):
|
||||
engine_client = cast(AsyncOmni, engine_client)
|
||||
result = None
|
||||
stage_list = getattr(engine_client, "stage_list", None)
|
||||
if isinstance(stage_list, list):
|
||||
default_params_list: list[OmniSamplingParams] | None = getattr(
|
||||
engine_client, "default_sampling_params_list", None
|
||||
normalized_stage_configs = list(stage_configs)
|
||||
if not normalized_stage_configs:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
|
||||
detail="Stage configs not found. Start server with a multi-stage omni model.",
|
||||
)
|
||||
if not isinstance(default_params_list, list):
|
||||
default_params_list = [
|
||||
OmniDiffusionSamplingParams() if st == "diffusion" else SamplingParams() for st in stage_types
|
||||
]
|
||||
else:
|
||||
default_params_list = list(default_params_list)
|
||||
if len(default_params_list) != len(stage_types):
|
||||
default_params_list = (
|
||||
default_params_list
|
||||
+ [OmniDiffusionSamplingParams() if st == "diffusion" else SamplingParams() for st in stage_types]
|
||||
)[: len(stage_types)]
|
||||
|
||||
sampling_params_list: list[OmniSamplingParams] = []
|
||||
for idx, stage_type in enumerate(stage_types):
|
||||
if stage_type == "diffusion":
|
||||
sampling_params_list.append(gen_params)
|
||||
else:
|
||||
base_params = default_params_list[idx]
|
||||
sampling_params_list.append(base_params)
|
||||
|
||||
async for output in engine_client.generate(
|
||||
sampling_params_list=sampling_params_list,
|
||||
**kwargs,
|
||||
):
|
||||
result = output
|
||||
default_params_list: list[OmniSamplingParams] | None = getattr(
|
||||
engine_client,
|
||||
"default_sampling_params_list",
|
||||
None,
|
||||
)
|
||||
if not isinstance(default_params_list, list):
|
||||
default_params_list = []
|
||||
else:
|
||||
result = await engine_client.generate(
|
||||
sampling_params_list=[gen_params],
|
||||
**kwargs,
|
||||
)
|
||||
default_params_list = list(default_params_list)
|
||||
|
||||
sampling_params_list: list[OmniSamplingParams] = []
|
||||
for idx, stage_cfg in enumerate(normalized_stage_configs):
|
||||
stage_type = get_stage_type(stage_cfg)
|
||||
if stage_type == "diffusion":
|
||||
sampling_params_list.append(gen_params)
|
||||
continue
|
||||
|
||||
if idx < len(default_params_list):
|
||||
default_stage_params = default_params_list[idx]
|
||||
else:
|
||||
default_stage_params = SamplingParams()
|
||||
|
||||
if hasattr(default_stage_params, "clone"):
|
||||
try:
|
||||
default_stage_params = default_stage_params.clone()
|
||||
except Exception:
|
||||
pass
|
||||
sampling_params_list.append(default_stage_params)
|
||||
|
||||
async for output in engine_client.generate(
|
||||
sampling_params_list=sampling_params_list,
|
||||
**kwargs,
|
||||
):
|
||||
result = output
|
||||
|
||||
if result is None:
|
||||
raise HTTPException(
|
||||
|
||||
Reference in new issue
Block a user