add precommit
This commit is contained in:
Gaohan123 committed
1 parent
1bf3424e96
commit
a04e945166
24 files changed
+434
-377
No files matched your search
@@ -73,20 +73,20 @@ from enum import Enum
|
||||
|
||||
class DiTCacheManager:
|
||||
"""Manages DiT cache for diffusion models."""
|
||||
|
||||
|
||||
def __init__(self, config: DiTCacheConfig):
|
||||
self.config = config
|
||||
self.cache = {}
|
||||
self.cache_stats = {}
|
||||
|
||||
|
||||
async def get_cache(self, cache_key: str) -> Optional[Any]:
|
||||
"""Retrieve cached data."""
|
||||
pass
|
||||
|
||||
|
||||
async def set_cache(self, cache_key: str, data: Any) -> None:
|
||||
"""Store data in cache."""
|
||||
pass
|
||||
|
||||
|
||||
async def invalidate_cache(self, cache_key: str) -> None:
|
||||
"""Invalidate cached data."""
|
||||
pass
|
||||
@@ -103,7 +103,7 @@ class OmniScheduler():
|
||||
self.scheduler = self._create_scheduler()
|
||||
self.cache_manager = DiTCacheManager(config.dit_cache_config)
|
||||
self._running = False
|
||||
|
||||
|
||||
def _create_scheduler(self) -> BaseScheduler:
|
||||
"""Factory method to create appropriate scheduler."""
|
||||
if self.config.scheduler_type == SchedulerType.FIFO:
|
||||
@@ -118,13 +118,13 @@ class OmniScheduler():
|
||||
async def schedule(self, request: OmniRequest) -> bool:
|
||||
"""
|
||||
Schedule a request for processing.
|
||||
|
||||
|
||||
Args:
|
||||
request: The request to schedule
|
||||
|
||||
|
||||
Returns:
|
||||
bool: True if successfully scheduled, False otherwise
|
||||
|
||||
|
||||
Raises:
|
||||
SchedulerError: If scheduling fails
|
||||
QueueFullError: If queue is at capacity
|
||||
@@ -137,7 +137,7 @@ class OmniScheduler():
|
||||
def update_config(self, new_config: SchedulerConfig) -> None:
|
||||
"""Update scheduler configuration."""
|
||||
pass
|
||||
|
||||
|
||||
def get_config(self) -> SchedulerConfig:
|
||||
"""Get current configuration."""
|
||||
pass
|
||||
@@ -149,12 +149,12 @@ class OmniScheduler():
|
||||
"""Start the scheduler."""
|
||||
self._running = True
|
||||
# Start background tasks
|
||||
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Stop the scheduler gracefully."""
|
||||
self._running = False
|
||||
# Cleanup resources
|
||||
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
"""Force shutdown the scheduler."""
|
||||
# Immediate cleanup
|
||||
@@ -170,7 +170,7 @@ class OmniScheduler():
|
||||
"processed_requests": self.scheduler.processed_count(),
|
||||
"cache_hit_rate": self.cache_manager.get_hit_rate()
|
||||
}
|
||||
|
||||
|
||||
def get_metrics(self) -> Dict[str, float]:
|
||||
"""Get performance metrics."""
|
||||
pass
|
||||
@@ -182,21 +182,21 @@ class OmniScheduler():
|
||||
@dataclass
|
||||
class OmniConfig:
|
||||
"""Configuration for the core module."""
|
||||
|
||||
|
||||
# Scheduler configuration
|
||||
scheduler: SchedulerConfig = field(default_factory=SchedulerConfig)
|
||||
|
||||
|
||||
# Cache configuration
|
||||
dit_cache: DiTCacheConfig = field(default_factory=DiTCacheConfig)
|
||||
|
||||
|
||||
# Resource limits
|
||||
max_memory_gb: float = 16.0
|
||||
max_gpu_utilization: float = 0.8
|
||||
|
||||
|
||||
# Timeouts
|
||||
request_timeout: int = 300
|
||||
worker_timeout: int = 60
|
||||
|
||||
|
||||
def validate(self) -> None:
|
||||
"""Validate configuration parameters."""
|
||||
if self.max_memory_gb <= 0:
|
||||
|
||||
+2
-2
@@ -157,10 +157,10 @@ Each API template follows this standardized structure:
|
||||
# Clone the repository
|
||||
git clone https://github.com/hsliuustc0106/vllm-omni.git
|
||||
cd vllm-omni
|
||||
|
||||
|
||||
# Install dependencies
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
|
||||
# Set up pre-commit hooks
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
@@ -52,7 +52,7 @@ According to analysis for current popular open-source models, most of them have
|
||||
* **Extension, multiple AR+DiT (e.g. Qwen-omni)**
|
||||
A new model for multimodal input and output. It designs the Thinker-Talker-Codec structure, which is a double-AR+DiT format.
|
||||
![][image5]
|
||||
|
||||
|
||||
|
||||
|
||||
# Architectural Overview
|
||||
@@ -112,8 +112,8 @@ vLLM-omni will import core modules from existing vLLM framework and introduce se
|
||||
|
||||
# Contact
|
||||
|
||||
For any questions or feedback regarding this design document, please contact
|
||||
Han Gao: [hgaoaf@connect.ust.hk](mailto:hgaoaf@connect.ust.hk),
|
||||
For any questions or feedback regarding this design document, please contact
|
||||
Han Gao: [hgaoaf@connect.ust.hk](mailto:hgaoaf@connect.ust.hk),
|
||||
Hongsheng Liu: [hsliuustc@gmail.com](mailto:hsliuustc@gmail.com)
|
||||
|
||||
[image1]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALYAAAFwCAYAAAAVNoEZAABxw0lEQVR4XuydBVRcWb6ve90709OStnTScel4x93dIUaEuBJ3dzciRPDgEtzdCRpCBCIEEkIgQHCCBHe+t6uSvNvNnZ55r2/fng5zvrX+69TZp4qkan3nz2/vKup88sm/OWqXL3HxwnnOnj7JubOnuaB6jqtXLqJ28TxXRWmrX8Ha0gIzE0OuX1bFQFsTcxMDLMyMMDXS44bWVRxsLLG5aYKuxlVsLMxwc3bAWF8bNzdnHGwt8fb2wNXZHlcnW3HMDjsbC4Ju3cLO2hwvT3ecnexwsLcmIMAHHy83+X18PVxxtLMUt+1xcbTBw8URRwdr+f1tLM1wd3PB08NZlAs24ue4uTrKf0bD5yfxb8qWDWvZtnE9W0Vt37yBvTu3cWDPTlE7OHviCBfPnsTEwECIfg7Vk0e5fkkVoxtamBneQF1NlSsXTmMrZJaN6elcF7LexFaIZ6SngbOjvfy2l5DUSwhoa2mK9U1jfL09ibwTKaS3IPx2BA4ONnh7uuHr4yUX1d/Pi0B/X1xd7PHz8cTT3UUc85SfIMHBQUJ4a3EfH/mJ4OsjThoXB/njPNydJbEl3rFisTKrli5i9fLFrF21jC0bVdi2aR07tqzn+KF9nD91HF1NDU4eO8CZIwe5fO4UBjoacpEvnDnG5fMnsDQ1Rk/rOje0rwmxLeQd3chAC0d7W6zMjfBwc8JHiGlmpIu5oS6hISHcibgtOrE99+4+wE50dR8hraeHm1zOW4G+BAb4yjtyYICf/FiAED0wMICw0DDxODvR8QNEBeLv7yO6txPeotN7e7lLYku8Y8akccycOhGlGVNRVprJsoXzWLlEmbUrl7Jv5xbOHD8sBD/ApnWrOLZvN2ePH0FDRJVronMf3rdDiH1aiK4p9s+JE0AWS6xE59bA2EAHO6ub3DTRw1UI7Ovjg/oVEWV01Llz567ouH6i+wbz8FEMtiKaBPr7yzuvn6+nED9I3rH9/bwJDgqUSy3r1OHh4YSEBOPu7io6/W1CQ0PFSeAvxHeTd3B/X19JbIl3TB4zjOnjRzJz8jiUFCazYPZ0li1QYrXo4nt3bObE4QPs3LIRlRWLObh7h9jfj5rqGdGtj7N/11Yh9hk0r14W+yfQ0bgiZLZC8/pljPR1ROwwwdxYF2d7GyGfN1cunkJfiB0h4oe3lweRkfeIjn4iF9vf1wcnkbX9fL0ICb4ll1kutrgtkzdEyB4ZGUlQ0C35Y+/cuSPv3oHimOxkkHX2W6KjN3x+Ev+mjBzQmzGD+zJxxCAmjxrC9AmjUBZyL1dWEkJvEF17G6tEB1++aB47Nq0Xsm+Vy71PbHdv2yTP4LL9Q3t3onVdTYhsyCXV02LyqCuyuS762texlk0oXV05e/IQetrqBInuK5v8RUc/JiLiDvYiisgmf7Y2N0UssZJv1a9fFQJ7yiUPkXXrsBAePHiAn+jMsghy//590dlDRRTxk8cW2VhoWLAktsQ7Bv7UleH9egi5+8kFl3Xw+TOnsmTBLDGpXMsOMaFcunCuiChKbF27ml1CbtnEcvt6FbnY504eE517uxB+CxpXLwlxNbhwXjbhvCGXWFOMmZsZibztwMmj+7khsniA6Kwurs4iSoi87OKCuuj4urqa3DQ3xdHBVt6pXZwd0NJQJ1AmbGiw6NC3xYkQLbK6lxA6WC55yP8VWxZrbhEWHiKJLfGOPl06MKhnVwb06Cy2XZg0aqiIJJNYNHcG61cvl2dtZXF78byZrF22iHUrl7B53WqRwxcI6dfLO/WW9Wvk3f3i2VOonj7BeRFT9HW0ULtwRoh/FIMb2hgZ6otMLuvql9klos3COYri3xrOiEH96dujK907d+Snrp3o37c3Gzdu4MKFCxjqG7Bn9y7uRt6Vd+gHUQ9xFsKHiMmnbEy29ReRRbaaEhYWJo8qDZ+fxL8pfbt2pF+3TqJ+ZHCvLiKODGX2tAly8VRWLJFPJBcoKcpFX6E8j9WL300slyrPkXd0WdfeIE4A2UqKbHlQtpJy7tQxtNWvckZILYspOprX0dFSZ8+2DezcqMKsCaOZOGwQvTp1oHPbVnRo2YL2rZrz/TdN+O6rJrRp8YMQvA+q5y9w9NARbgUFEf3wEffuR2FrZyUXOiIiQnTyEPmkUSa2LNLcu/dAElviHb06taF357bIBB/cqzNjh/YXOXs0cxWnsWjebPlKyVwRTebNmILyLAWWzJ3NMuW5LJijIO/o61YvY/WShWxUWcmB3dvZLgQ/emAvZ04cZc/2zULmTaiKuLJv22ZmjxvJqH496dK6pagW9Gnfhp7tWtH5h+9p//23tPruG1qI+v6br/n2qy/47K9/odn337FGRQUHJ2fCbt/GyFhPxI4Q+eRRJrivr7d8AvngQZTo6tGS2BLv6P6jELtLO7nY/bv/yPCBPZkiBJylMIV5QuS5M6czR3EKStMnMU8m+5yZLFGazdwZU1mzfAmrly2UTyzXr1omjyMb1qxgn8jch/fvYdsGFXZsXMMusV0i7j+6V1eGdm3PGBF5pvXtztxBPZjetyujxL8/VPw/erRuzo9C8pZNheBNv+b7r7/iy88/o2PHjixcspTbtyO4IbK4bMnvzp13KySyzB0cIptYRktiS/wXnzdpzpdNmtGtQyt+6tiK/j06MXbEYCaPG8WMKROYOWUc0yaOY6ro4orittKMacxWmMysaeNZtmAei+fPEZ19FquWLmSdkHv5ovnyzL1dxJQ1Qvr1ohaOG8rxJTO4ukYJ/a1LODprDEsGdmdkux8Y2KopHb/+knZffk6LJp/xvRD5e3G7+Zdf0PKbr2gtJG8jZP/uqy9Zu24tqmdOEn77jrxjy96wkb1BExERyd2794m4c1cSW+Idn33ZlL99IaJA6x/o1KaZ6N7tGTmkP+NGDpXLPG38KLnkk8eNYPqkMXKpZ0ydIKQfK2LJHHkHV1aaIYReIM/jMtFlgss691KRzeeKn6G7YwVup3fgr3YAw83KLO/VkbX9OrNhYBfmd23JrC6tmNT+B7p89zVdRafu1uwbOjf9ivbffUUHsS/r4h3F/69Fs2YsWbhAZOn78owdGBgol1vWyWWTyYg7EZLYEu9o/cVXrO71E6aL53Nx1mT6iLw9sE9Phg/qx/jhQ+Q1doTYjhryXu6xTJNtx4scrjCVOSKiKImoIsvispJ19MULlEQGn8GsYf05vWI2+aGOZHqb4LJvNY+v7iP2+hEeXdpL2NF1uG9V5ti4nuwb1pUlPVoz88cWTGzblHGihrVuypA23zOsfXOGCPE7t/qBZt9+g7qmFmFh4XKxZXlb9kbPbdG1ZZ87afj8JP5NGd2mFfuHD8BCeQ43F8/hzMxxzB3al5H9fmJo/z4MH9CXEYMHMEpMKscNFxFl9AjGjxjERNHRFSbIJB/N9MljmDV9MjOnTUJh8nhmT5/CxNHDMNu/luI77pRGB5Dhb0t1zB2qnoRREuxEnpsx989sI3T/apzXKXBz6WRMl01Bf9F4VKcP5uj4Pmwd1p21g7ui3Lsj83p1oEerZrQV3fv7pt9y4uRxbgUHERIWRlh4BMFC9FvBoZLYEu/YObgfp0YO4ZLowtemjhViKXB1wVT2KIxBccRAxorOPVgIPnRQX8YMHSjv4GPeSz5p7EgmjhkhjywyoadPkmXxMfJMPmHoAOJstKhLjKLiaTiVCQ+pSY6jJj6K0jA3CjxNSdI/S4L2MR5c2sW9CzsIOqKC156lGC2bivr8MRye0I+dI3uyelBXlvXvxMA2zWnb7Ds+//QvjBkziiAxaQwJDRWTx3D5kqC/9Ja6xAcU27Rkcgvx67/59yzo2JrDo/thvVYZj51rCT20leCDm7DfvALN5XNRHj2UcQP7MKRPV9HJe4suLuvkQvZRw+SCTxg9XH57pDgZji6bSXVcJHWZidS+SaMuN43alGdysasfBlER6UNpsCNv/azJctQh1eoacdrHuX9hF67bFmK+SpFz04dwZHw/to3sxfqh3ZgssniXZl/LlwT/+p//ia+vn5g8+uPt7SX/YJSrq4sktsQ7un39Fd2/+ZoB337NpJZNWdezI/pKY3FSmU+o2mVCjc3xd/LCw9kHbSNrDl/QQFFpAaNHDGNov3dZfIzo3rLJ5tiR7/L4zJGDCBQ5ui4tntr8TOpF1WWnUJsUQ+2z+1RHBVF1L4DSUFeKAuzJdjEkzUaT+BtniLq8F+ftypitVOCsEPvoxH7sHt2XzUN7MLtHGwa0aEpLIfYXf/sMS0sLPDw8cHR0xN7ORl4Nn5/Evymd2rXgxzY/0EF0wj5dO3NE2xTj5CLMC+rQL6zDtKQe+/I6rEvrMS6owfBNFQY5leimlnLI0pOl2/aLDN6X0UPede9TG1dxS/sMiO5cHO5DcYgHed4OZNkZURbqSfWDYKrvBlN+25/iIDcK/Rx542JEpp0uTzWPEXl+Jw6b52O8fDrnFYZxcupg9o7pK7p2Dxb3aYdil5a0+eYrvvnicw4e3I/lTQsM9G+gr6eL3g1dSWyJd3T44Rt+bN2C+bsOc9w7ApMiuFkMN/JrsSyuwbm8FuuiWkzF/s3Caizf1mCZX4VpbgV6yW+5/iCRpVv3MkFMJBdMn0BehDf50SGQLaLH6xfUy+JHVCiVYZ6UeltRGexO9T2xHxFI0S2RtX0cyHMxI9P2Bo9Flw8/vQ3bjfMwWDqVc0LsE1MGsXd0H7aP6M6SPu2Z3bUVbb9pwjdffsnKlcvQ1rjO1SuXuHZVVpclsf8gNomSvdgfKuaXh//1qD3O4fLrKjRya9HJrUE3uxqr4mrcy2vwLKvDMKsKSyG0s9i3Lxad+20txqJj3xRlJeQ2yyhBNy6Dq0GPWLN8Kbn3g6jNEdEjL5uHphZcO3QRNyNHEnyCqY2NpfbFC2rjnlBxL0x0clkMuUmSmQZPdS9wR3U/t4Line truncated
|
||||
|
||||
@@ -136,7 +136,7 @@ class OmniLLM(LLM):
|
||||
self.engine_list: List[LLMEngine] = []
|
||||
self.output_processor = MultimodalOutputProcessor()
|
||||
self._initialize_stage_engines()
|
||||
|
||||
|
||||
def _initialize_stage_engines(self) -> None:
|
||||
"""Initialize LLMEngine instances for each stage"""
|
||||
for stage_config in self.stage_configs:
|
||||
@@ -145,24 +145,24 @@ class OmniLLM(LLM):
|
||||
elif stage_config.engine_type == "DiT":
|
||||
engine = self._create_dit_engine(stage_config)
|
||||
self.engine_list.append(engine)
|
||||
|
||||
|
||||
def generate(self, stage_args_list: List[Dict], **kwargs) -> List[RequestOutput]:
|
||||
"""Main generation interface - orchestrates multi-stage processing"""
|
||||
current_output = None
|
||||
|
||||
|
||||
for i, (stage_config, stage_args) in enumerate(zip(self.stage_configs, stage_args_list)):
|
||||
stage_engine = self.engine_list[i]
|
||||
|
||||
|
||||
# Prepare input for this stage
|
||||
processed_input = self._process_stage_inputs(stage_config, stage_args, current_output)
|
||||
|
||||
|
||||
# Execute stage
|
||||
stage_output = self._execute_stage(stage_engine, processed_input)
|
||||
|
||||
|
||||
# Update for next stage
|
||||
current_output = stage_output
|
||||
stage_config.stage_output = stage_output
|
||||
|
||||
|
||||
# Process final output
|
||||
return self.output_processor.process_output(current_output)
|
||||
```
|
||||
@@ -178,7 +178,7 @@ class AsyncOmniLLM(AsyncLLM):
|
||||
self.async_engine_list: List[AsyncLLM] = []
|
||||
self.output_processor = MultimodalOutputProcessor()
|
||||
self._initialize_async_stage_engines()
|
||||
|
||||
|
||||
async def generate_async(self, stage_args_list: List[Dict], **kwargs) -> List[RequestOutput]:
|
||||
"""Async generation interface"""
|
||||
# Similar to OmniLLM but with async/await patterns
|
||||
@@ -241,7 +241,7 @@ class MultimodalOutputProcessor(OutputProcessor):
|
||||
"latents": self._process_latents_output,
|
||||
"text": self._process_text_output,
|
||||
}
|
||||
|
||||
|
||||
def process_outputs(self, engine_core_outputs: List[EngineCoreOutput], ...):
|
||||
"""Process multimodal outputs based on type"""
|
||||
for engine_core_output in engine_core_outputs:
|
||||
|
||||
Reference in new issue
Block a user