""" Omni serve command for vLLM-omni. """ import argparse import uvloop from vllm.entrypoints.cli.types import CLISubcommand from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG from vllm.logger import init_logger from vllm.utils import FlexibleArgumentParser from vllm_omni.entrypoints.openai.api_server import omni_run_server logger = init_logger(__name__) DESCRIPTION = """Launch a local OpenAI-compatible API server to serve Omni LLM completions via HTTP. Defaults to Qwen/Qwen2.5-Omni-7B if no model is specified. Search by using: `--help=` to explore options by section (e.g., --help=ModelConfig, --help=Frontend) Use `--help=all` to show all available flags at once. """ class OmniServeCommand(CLISubcommand): """The `serve` subcommand for the vLLM CLI.""" name = "serve" @staticmethod def cmd(args: argparse.Namespace) -> None: # If model is specified in CLI (as positional arg), it takes precedence if hasattr(args, "model_tag") and args.model_tag is not None: args.model = args.model_tag uvloop.run(omni_run_server(args)) def validate(self, args: argparse.Namespace) -> None: validate_parsed_serve_args(args) def subparser_init(self, subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser: serve_parser = subparsers.add_parser( self.name, description=DESCRIPTION, usage="vllm serve [model_tag] --omni [options]", ) serve_parser = make_arg_parser(serve_parser) serve_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name) serve_parser.add_argument( "--omni", action="store_true", help="Enable vLLM-omni mode", ) serve_parser.add_argument( "--stage-configs-path", type=str, default=None, help="Path to the stage configs file. If not specified, the stage configs will be loaded from the model.", ) return serve_parser def cmd_init() -> list[CLISubcommand]: return [OmniServeCommand()]