add precommit
This commit is contained in:
1 parent
1bf3424e96
commit
a04e945166
24 files changed
+434
-377
No files matched your search
@@ -3,8 +3,7 @@ CLI entry point for vLLM-omni that intercepts vLLM commands.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import argparse
|
||||
from typing import List, Optional
|
||||
|
||||
from vllm_omni.entrypoints.cli.serve import OmniServeCommand
|
||||
|
||||
|
||||
@@ -19,6 +18,7 @@ def main():
|
||||
else:
|
||||
# Forward to original vLLM CLI
|
||||
from vllm.entrypoints.cli.main import main as vllm_main
|
||||
|
||||
vllm_main()
|
||||
|
||||
|
||||
|
||||
@@ -4,125 +4,111 @@ Omni serve command for vLLM-omni.
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from typing import List, Optional
|
||||
from typing import List
|
||||
|
||||
from vllm_omni.config import DiTConfig, create_ar_stage_config, create_dit_stage_config
|
||||
|
||||
from ..omni_llm import AsyncOmniLLM
|
||||
from vllm_omni.config import (
|
||||
DiTConfig,
|
||||
create_ar_stage_config,
|
||||
create_dit_stage_config,
|
||||
)
|
||||
|
||||
|
||||
class OmniServeCommand:
|
||||
"""Command handler for vLLM-omni serve command."""
|
||||
|
||||
|
||||
def __init__(self):
|
||||
self.parser = self._create_parser()
|
||||
|
||||
|
||||
def _create_parser(self) -> argparse.ArgumentParser:
|
||||
"""Create argument parser for omni serve command."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="vLLM-omni: Multi-modality models inference and serving"
|
||||
)
|
||||
|
||||
|
||||
# Model arguments - make it optional with default
|
||||
parser.add_argument(
|
||||
"model",
|
||||
nargs="?",
|
||||
default="Qwen/Qwen3-0.6B",
|
||||
help="Path to the model or model name (default: Qwen/Qwen3-0.6B)"
|
||||
help="Path to the model or model name (default: Qwen/Qwen3-0.6B)",
|
||||
)
|
||||
|
||||
|
||||
# Server arguments
|
||||
parser.add_argument(
|
||||
"--port",
|
||||
type=int,
|
||||
default=8000,
|
||||
help="Port to run the server on"
|
||||
help="Port to run the server on",
|
||||
)
|
||||
|
||||
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type=str,
|
||||
default="0.0.0.0",
|
||||
help="Host to run the server on"
|
||||
help="Host to run the server on",
|
||||
)
|
||||
|
||||
|
||||
# Stage configuration arguments
|
||||
parser.add_argument(
|
||||
"--ar-stage",
|
||||
type=str,
|
||||
help="AR stage model path"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--dit-stage",
|
||||
type=str,
|
||||
help="DiT stage model path"
|
||||
)
|
||||
|
||||
parser.add_argument("--ar-stage", type=str, help="AR stage model path")
|
||||
|
||||
parser.add_argument("--dit-stage", type=str, help="DiT stage model path")
|
||||
|
||||
parser.add_argument(
|
||||
"--dit-steps",
|
||||
type=int,
|
||||
default=50,
|
||||
help="Number of DiT inference steps"
|
||||
help="Number of DiT inference steps",
|
||||
)
|
||||
|
||||
|
||||
parser.add_argument(
|
||||
"--dit-guidance-scale",
|
||||
type=float,
|
||||
default=7.5,
|
||||
help="DiT guidance scale"
|
||||
help="DiT guidance scale",
|
||||
)
|
||||
|
||||
|
||||
parser.add_argument(
|
||||
"--use-diffusers",
|
||||
action="store_true",
|
||||
help="Use diffusers pipeline for DiT stage"
|
||||
help="Use diffusers pipeline for DiT stage",
|
||||
)
|
||||
|
||||
|
||||
# Other arguments
|
||||
parser.add_argument(
|
||||
"--log-stats",
|
||||
action="store_true",
|
||||
help="Enable logging statistics"
|
||||
"--log-stats", action="store_true", help="Enable logging statistics"
|
||||
)
|
||||
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def run(self, args: List[str]) -> None:
|
||||
"""Run the omni serve command."""
|
||||
parsed_args = self.parser.parse_args(args)
|
||||
|
||||
|
||||
# Create stage configurations
|
||||
stage_configs = self._create_stage_configs(parsed_args)
|
||||
|
||||
|
||||
# Create AsyncOmniLLM instance
|
||||
omni_llm = AsyncOmniLLM(
|
||||
stage_configs=stage_configs,
|
||||
log_stats=parsed_args.log_stats
|
||||
stage_configs=stage_configs, log_stats=parsed_args.log_stats
|
||||
)
|
||||
|
||||
|
||||
# Start the server
|
||||
asyncio.run(self._start_server(omni_llm, parsed_args))
|
||||
|
||||
|
||||
def _create_stage_configs(self, args) -> List:
|
||||
"""Create stage configurations based on arguments."""
|
||||
stage_configs = []
|
||||
stage_id = 0
|
||||
|
||||
|
||||
# Add AR stage - use main model if ar_stage not specified
|
||||
ar_model = args.ar_stage if args.ar_stage else args.model
|
||||
ar_config = create_ar_stage_config(
|
||||
stage_id=stage_id,
|
||||
model_path=ar_model,
|
||||
input_modalities=["text"],
|
||||
output_modalities=["text"]
|
||||
output_modalities=["text"],
|
||||
)
|
||||
stage_configs.append(ar_config)
|
||||
stage_id += 1
|
||||
|
||||
|
||||
# Add DiT stage if specified
|
||||
if args.dit_stage:
|
||||
dit_model = args.dit_stage
|
||||
@@ -131,7 +117,7 @@ class OmniServeCommand:
|
||||
dit_model = "stabilityai/stable-diffusion-2-1"
|
||||
else:
|
||||
dit_model = None
|
||||
|
||||
|
||||
if dit_model:
|
||||
dit_config = DiTConfig(
|
||||
num_inference_steps=args.dit_steps,
|
||||
@@ -139,17 +125,17 @@ class OmniServeCommand:
|
||||
use_diffusers=args.use_diffusers,
|
||||
diffusers_pipeline="auto" if args.use_diffusers else None,
|
||||
)
|
||||
|
||||
|
||||
dit_stage_config = create_dit_stage_config(
|
||||
stage_id=stage_id,
|
||||
model_path=dit_model,
|
||||
input_modalities=["text"],
|
||||
output_modalities=["image"],
|
||||
dit_config=dit_config
|
||||
dit_config=dit_config,
|
||||
)
|
||||
stage_configs.append(dit_stage_config)
|
||||
stage_id += 1
|
||||
|
||||
|
||||
# If no specific stages are specified, use the main model
|
||||
if not stage_configs:
|
||||
# Try to detect if it's a multimodal model
|
||||
@@ -159,23 +145,23 @@ class OmniServeCommand:
|
||||
stage_id=0,
|
||||
model_path=args.model,
|
||||
input_modalities=["text"],
|
||||
output_modalities=["text"]
|
||||
output_modalities=["text"],
|
||||
)
|
||||
stage_configs.append(ar_config)
|
||||
|
||||
|
||||
dit_config = DiTConfig(
|
||||
num_inference_steps=args.dit_steps,
|
||||
guidance_scale=args.dit_guidance_scale,
|
||||
use_diffusers=args.use_diffusers,
|
||||
diffusers_pipeline="auto" if args.use_diffusers else None,
|
||||
)
|
||||
|
||||
|
||||
dit_stage_config = create_dit_stage_config(
|
||||
stage_id=1,
|
||||
model_path=args.model,
|
||||
input_modalities=["text"],
|
||||
output_modalities=["image"],
|
||||
dit_config=dit_config
|
||||
dit_config=dit_config,
|
||||
)
|
||||
stage_configs.append(dit_stage_config)
|
||||
else:
|
||||
@@ -184,23 +170,19 @@ class OmniServeCommand:
|
||||
stage_id=0,
|
||||
model_path=args.model,
|
||||
input_modalities=["text"],
|
||||
output_modalities=["text"]
|
||||
output_modalities=["text"],
|
||||
)
|
||||
stage_configs.append(ar_config)
|
||||
|
||||
|
||||
return stage_configs
|
||||
|
||||
|
||||
async def _start_server(self, omni_llm: AsyncOmniLLM, args) -> None:
|
||||
"""Start the API server."""
|
||||
try:
|
||||
# Import here to avoid circular imports
|
||||
from vllm_omni.entrypoints.api_server import run_server
|
||||
|
||||
await run_server(
|
||||
omni_llm_instance=omni_llm,
|
||||
host=args.host,
|
||||
port=args.port
|
||||
)
|
||||
|
||||
await run_server(omni_llm_instance=omni_llm, host=args.host, port=args.port)
|
||||
except KeyboardInterrupt:
|
||||
print("\nShutting down server...")
|
||||
except Exception as e:
|
||||
|
||||
Reference in new issue
Block a user