vllm-omni/vllm_omni/entrypoints/cli/serve.py

"""
Omni serve command for vLLM-omni.
"""

import argparse
import asyncio
from typing import List, Optional

from ..omni_llm import AsyncOmniLLM
from vllm_omni.config import (
    DiTConfig,
    create_ar_stage_config,
    create_dit_stage_config,
)


class OmniServeCommand:
    """Command handler for vLLM-omni serve command."""
    
    def __init__(self):
        self.parser = self._create_parser()
    
    def _create_parser(self) -> argparse.ArgumentParser:
        """Create argument parser for omni serve command."""
        parser = argparse.ArgumentParser(
            description="vLLM-omni: Multi-modality models inference and serving"
        )
        
        # Model arguments - make it optional with default
        parser.add_argument(
            "model",
            nargs="?",
            default="Qwen/Qwen3-0.6B",
            help="Path to the model or model name (default: Qwen/Qwen3-0.6B)"
        )
        
        # Server arguments
        parser.add_argument(
            "--port",
            type=int,
            default=8000,
            help="Port to run the server on"
        )
        
        parser.add_argument(
            "--host",
            type=str,
            default="0.0.0.0",
            help="Host to run the server on"
        )
        
        # Stage configuration arguments
        parser.add_argument(
            "--ar-stage",
            type=str,
            help="AR stage model path"
        )
        
        parser.add_argument(
            "--dit-stage",
            type=str,
            help="DiT stage model path"
        )
        
        parser.add_argument(
            "--dit-steps",
            type=int,
            default=50,
            help="Number of DiT inference steps"
        )
        
        parser.add_argument(
            "--dit-guidance-scale",
            type=float,
            default=7.5,
            help="DiT guidance scale"
        )
        
        parser.add_argument(
            "--use-diffusers",
            action="store_true",
            help="Use diffusers pipeline for DiT stage"
        )
        
        # Other arguments
        parser.add_argument(
            "--log-stats",
            action="store_true",
            help="Enable logging statistics"
        )
        
        return parser
    
    def run(self, args: List[str]) -> None:
        """Run the omni serve command."""
        parsed_args = self.parser.parse_args(args)
        
        # Create stage configurations
        stage_configs = self._create_stage_configs(parsed_args)
        
        # Create AsyncOmniLLM instance
        omni_llm = AsyncOmniLLM(
            stage_configs=stage_configs,
            log_stats=parsed_args.log_stats
        )
        
        # Start the server
        asyncio.run(self._start_server(omni_llm, parsed_args))
    
    def _create_stage_configs(self, args) -> List:
        """Create stage configurations based on arguments."""
        stage_configs = []
        stage_id = 0
        
        # Add AR stage - use main model if ar_stage not specified
        ar_model = args.ar_stage if args.ar_stage else args.model
        ar_config = create_ar_stage_config(
            stage_id=stage_id,
            model_path=ar_model,
            input_modalities=["text"],
            output_modalities=["text"]
        )
        stage_configs.append(ar_config)
        stage_id += 1
        
        # Add DiT stage if specified
        if args.dit_stage:
            dit_model = args.dit_stage
        elif args.use_diffusers:
            # Use a default DiT model if diffusers is enabled
            dit_model = "stabilityai/stable-diffusion-2-1"
        else:
            dit_model = None
            
        if dit_model:
            dit_config = DiTConfig(
                num_inference_steps=args.dit_steps,
                guidance_scale=args.dit_guidance_scale,
                use_diffusers=args.use_diffusers,
                diffusers_pipeline="auto" if args.use_diffusers else None,
            )
            
            dit_stage_config = create_dit_stage_config(
                stage_id=stage_id,
                model_path=dit_model,
                input_modalities=["text"],
                output_modalities=["image"],
                dit_config=dit_config
            )
            stage_configs.append(dit_stage_config)
            stage_id += 1
        
        # If no specific stages are specified, use the main model
        if not stage_configs:
            # Try to detect if it's a multimodal model
            if "omni" in args.model.lower() or "multimodal" in args.model.lower():
                # Assume it's a multimodal model that can handle both AR and DiT
                ar_config = create_ar_stage_config(
                    stage_id=0,
                    model_path=args.model,
                    input_modalities=["text"],
                    output_modalities=["text"]
                )
                stage_configs.append(ar_config)
                
                dit_config = DiTConfig(
                    num_inference_steps=args.dit_steps,
                    guidance_scale=args.dit_guidance_scale,
                    use_diffusers=args.use_diffusers,
                    diffusers_pipeline="auto" if args.use_diffusers else None,
                )
                
                dit_stage_config = create_dit_stage_config(
                    stage_id=1,
                    model_path=args.model,
                    input_modalities=["text"],
                    output_modalities=["image"],
                    dit_config=dit_config
                )
                stage_configs.append(dit_stage_config)
            else:
                # Default to AR stage
                ar_config = create_ar_stage_config(
                    stage_id=0,
                    model_path=args.model,
                    input_modalities=["text"],
                    output_modalities=["text"]
                )
                stage_configs.append(ar_config)
        
        return stage_configs
    
    async def _start_server(self, omni_llm: AsyncOmniLLM, args) -> None:
        """Start the API server."""
        try:
            # Import here to avoid circular imports
            from vllm_omni.entrypoints.api_server import run_server
            
            await run_server(
                omni_llm_instance=omni_llm,
                host=args.host,
                port=args.port
            )
        except KeyboardInterrupt:
            print("\nShutting down server...")
        except Exception as e:
            print(f"Error starting server: {e}")
            raise