[Entrypoints] Support Online Serving for Diffusion-only Models (#259)

Signed-off-by: Chenguang ZHENG <645327136@qq.com>
Signed-off-by: yinpe <11810305@mail.sustech.edu.cn>
Co-authored-by: yinpe <11810305@mail.sustech.edu.cn>
This commit is contained in:
Chenguang Zhengandyinpe authored and GitHub committed 2025-12-17 17:40:45 +08:00
1 parent 8722c6374c
commit acf66ee2cc
19 files changed
+2506 -20

No files matched your search

+72 -3
View File
@@ -1,5 +1,8 @@
"""
Omni serve command for vLLM-Omni.
Supports both multi-stage LLM models (e.g., Qwen2.5-Omni) and
diffusion models (e.g., Qwen-Image) through the same CLI interface.
"""
import argparse
@@ -15,8 +18,19 @@ from vllm_omni.entrypoints.openai.api_server import omni_run_server
logger = init_logger(__name__)
DESCRIPTION = """Launch a local OpenAI-compatible API server to serve Omni LLM
completions via HTTP. Defaults to Qwen/Qwen2.5-Omni-7B if no model is specified.
DESCRIPTION = """Launch a local OpenAI-compatible API server to serve Omni models
via HTTP. Supports both multi-stage LLM models and diffusion models.
The server automatically detects the model type:
- LLM models: Served via /v1/chat/completions endpoint
- Diffusion models: Served via /v1/images/generations endpoint
Examples:
# Start an Omni LLM server
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
# Start a diffusion model server
vllm serve Qwen/Qwen-Image --omni --port 8091
Search by using: `--help=<ConfigGroup>` to explore options by section (e.g.,
--help=ModelConfig, --help=Frontend)
@@ -38,6 +52,13 @@ class OmniServeCommand(CLISubcommand):
uvloop.run(omni_run_server(args))
def validate(self, args: argparse.Namespace) -> None:
# Skip validation for diffusion models as they have different requirements
from vllm_omni.diffusion.utils.hf_utils import is_diffusion_model
model = getattr(args, "model_tag", None) or getattr(args, "model", None)
if model and is_diffusion_model(model):
logger.info("Detected diffusion model: %s", model)
return
validate_parsed_serve_args(args)
def subparser_init(self, subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
@@ -52,7 +73,7 @@ class OmniServeCommand(CLISubcommand):
serve_parser.add_argument(
"--omni",
action="store_true",
help="Enable vLLM-Omni mode",
help="Enable vLLM-Omni mode for multi-modal and diffusion models",
)
serve_parser.add_argument(
"--stage-configs-path",
@@ -108,6 +129,54 @@ class OmniServeCommand(CLISubcommand):
default=None,
help="The address of the Ray cluster to connect to.",
)
# Diffusion model specific arguments
serve_parser.add_argument(
"--num-gpus",
type=int,
default=1,
help="Number of GPUs to use for diffusion model inference.",
)
# Cache optimization parameters
serve_parser.add_argument(
"--cache-backend",
type=str,
default="none",
help="Cache backend for diffusion models, options: 'tea_cache', 'cache_dit'",
)
serve_parser.add_argument(
"--cache-config",
type=str,
default=None,
help="JSON string of cache configuration (e.g., '{\"rel_l1_thresh\": 0.2}').",
)
# VAE memory optimization parameters
serve_parser.add_argument(
"--vae-use-slicing",
action="store_true",
help="Enable VAE slicing for memory optimization (useful for mitigating OOM issues).",
)
serve_parser.add_argument(
"--vae-use-tiling",
action="store_true",
help="Enable VAE tiling for memory optimization (useful for mitigating OOM issues).",
)
# Video model parameters (e.g., Wan2.2) - engine-level
serve_parser.add_argument(
"--boundary-ratio",
type=float,
default=None,
help="Boundary split ratio for low/high DiT in video models (e.g., 0.875 for Wan2.2).",
)
serve_parser.add_argument(
"--flow-shift",
type=float,
default=None,
help="Scheduler flow_shift for video models (e.g., 5.0 for 720p, 12.0 for 480p).",
)
return serve_parser