[Entrypoints] Support Online Serving for Diffusion-only Models (#259)
Signed-off-by: Chenguang ZHENG <645327136@qq.com> Signed-off-by: yinpe <11810305@mail.sustech.edu.cn> Co-authored-by: yinpe <11810305@mail.sustech.edu.cn>
This commit is contained in:
1 parent
8722c6374c
commit
acf66ee2cc
19 files changed
+2506
-20
No files matched your search
@@ -1,5 +1,8 @@
|
||||
"""
|
||||
Omni serve command for vLLM-Omni.
|
||||
|
||||
Supports both multi-stage LLM models (e.g., Qwen2.5-Omni) and
|
||||
diffusion models (e.g., Qwen-Image) through the same CLI interface.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -15,8 +18,19 @@ from vllm_omni.entrypoints.openai.api_server import omni_run_server
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
DESCRIPTION = """Launch a local OpenAI-compatible API server to serve Omni LLM
|
||||
completions via HTTP. Defaults to Qwen/Qwen2.5-Omni-7B if no model is specified.
|
||||
DESCRIPTION = """Launch a local OpenAI-compatible API server to serve Omni models
|
||||
via HTTP. Supports both multi-stage LLM models and diffusion models.
|
||||
|
||||
The server automatically detects the model type:
|
||||
- LLM models: Served via /v1/chat/completions endpoint
|
||||
- Diffusion models: Served via /v1/images/generations endpoint
|
||||
|
||||
Examples:
|
||||
# Start an Omni LLM server
|
||||
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
|
||||
|
||||
# Start a diffusion model server
|
||||
vllm serve Qwen/Qwen-Image --omni --port 8091
|
||||
|
||||
Search by using: `--help=<ConfigGroup>` to explore options by section (e.g.,
|
||||
--help=ModelConfig, --help=Frontend)
|
||||
@@ -38,6 +52,13 @@ class OmniServeCommand(CLISubcommand):
|
||||
uvloop.run(omni_run_server(args))
|
||||
|
||||
def validate(self, args: argparse.Namespace) -> None:
|
||||
# Skip validation for diffusion models as they have different requirements
|
||||
from vllm_omni.diffusion.utils.hf_utils import is_diffusion_model
|
||||
|
||||
model = getattr(args, "model_tag", None) or getattr(args, "model", None)
|
||||
if model and is_diffusion_model(model):
|
||||
logger.info("Detected diffusion model: %s", model)
|
||||
return
|
||||
validate_parsed_serve_args(args)
|
||||
|
||||
def subparser_init(self, subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
|
||||
@@ -52,7 +73,7 @@ class OmniServeCommand(CLISubcommand):
|
||||
serve_parser.add_argument(
|
||||
"--omni",
|
||||
action="store_true",
|
||||
help="Enable vLLM-Omni mode",
|
||||
help="Enable vLLM-Omni mode for multi-modal and diffusion models",
|
||||
)
|
||||
serve_parser.add_argument(
|
||||
"--stage-configs-path",
|
||||
@@ -108,6 +129,54 @@ class OmniServeCommand(CLISubcommand):
|
||||
default=None,
|
||||
help="The address of the Ray cluster to connect to.",
|
||||
)
|
||||
|
||||
# Diffusion model specific arguments
|
||||
serve_parser.add_argument(
|
||||
"--num-gpus",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of GPUs to use for diffusion model inference.",
|
||||
)
|
||||
|
||||
# Cache optimization parameters
|
||||
serve_parser.add_argument(
|
||||
"--cache-backend",
|
||||
type=str,
|
||||
default="none",
|
||||
help="Cache backend for diffusion models, options: 'tea_cache', 'cache_dit'",
|
||||
)
|
||||
serve_parser.add_argument(
|
||||
"--cache-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help="JSON string of cache configuration (e.g., '{\"rel_l1_thresh\": 0.2}').",
|
||||
)
|
||||
|
||||
# VAE memory optimization parameters
|
||||
serve_parser.add_argument(
|
||||
"--vae-use-slicing",
|
||||
action="store_true",
|
||||
help="Enable VAE slicing for memory optimization (useful for mitigating OOM issues).",
|
||||
)
|
||||
serve_parser.add_argument(
|
||||
"--vae-use-tiling",
|
||||
action="store_true",
|
||||
help="Enable VAE tiling for memory optimization (useful for mitigating OOM issues).",
|
||||
)
|
||||
|
||||
# Video model parameters (e.g., Wan2.2) - engine-level
|
||||
serve_parser.add_argument(
|
||||
"--boundary-ratio",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Boundary split ratio for low/high DiT in video models (e.g., 0.875 for Wan2.2).",
|
||||
)
|
||||
serve_parser.add_argument(
|
||||
"--flow-shift",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Scheduler flow_shift for video models (e.g., 5.0 for 720p, 12.0 for 480p).",
|
||||
)
|
||||
return serve_parser
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user