hsliuustc0106
f4394f1d3d
Models whose request geometry opts out of the generic dummy run (H3: t2va needs an explicit aspect ratio and a bounded duration) can declare a dlo_dummy_run_recipe. Under distributed layerwise offload the engine's startup warmup runs it, so the learned cache-retention budget is primed before real traffic instead of on the first request. H3 declares a maximum-duration t2va generation: duration is the only workload variable at the fixed 768 short edge, so the learned peak dominates every production request and no real request re-learns. Validated on L20X: all cold-start releases happen during the startup warmup, zero during requests; first request 15.91 s vs 16.7 s unwarmed, steady 14.9 s; startup gains one max-length generation (~10-20 s). Non-DLO deployments keep the opted-out generic warmup unchanged. Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
77 lines
3.2 KiB
Python
77 lines
3.2 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from __future__ import annotations
|
|
|
|
import inspect
|
|
from typing import Any
|
|
|
|
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
|
from vllm_omni.diffusion.model_metadata import get_diffusion_model_metadata
|
|
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import resolve_diffusers_pipeline_class
|
|
from vllm_omni.diffusion.registry import DiffusionModelRegistry
|
|
|
|
|
|
def supports_multimodal_input(od_config: OmniDiffusionConfig) -> tuple[bool, bool]:
|
|
if (
|
|
od_config.diffusion_load_format == "diffusers"
|
|
and (pipe_cls := resolve_diffusers_pipeline_class(od_config)) is not None
|
|
):
|
|
signature = inspect.signature(pipe_cls.__call__)
|
|
support_image_input = "image" in signature.parameters
|
|
support_audio_input = (
|
|
"audio" in signature.parameters or "audio_latents" in signature.parameters
|
|
) # ref. LTX-2 format
|
|
return support_image_input, support_audio_input
|
|
|
|
supports_image_input = False
|
|
supports_audio_input = False
|
|
|
|
model_cls = DiffusionModelRegistry._try_load_model_cls(od_config.model_class_name)
|
|
if model_cls is not None:
|
|
supports_image_input = bool(getattr(model_cls, "support_image_input", False))
|
|
supports_audio_input = bool(getattr(model_cls, "support_audio_input", False))
|
|
return supports_image_input, supports_audio_input
|
|
|
|
|
|
def image_color_format(model_class_name: str) -> str:
|
|
model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name)
|
|
return getattr(model_cls, "color_format", "RGB")
|
|
|
|
|
|
def supports_audio_output(model_class_name: str) -> bool:
|
|
model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name)
|
|
if model_cls is None:
|
|
return False
|
|
return bool(getattr(model_cls, "support_audio_output", False))
|
|
|
|
|
|
def get_dummy_run_num_frames(model_class_name: str, supports_audio_input: bool) -> int:
|
|
"""Get num_frames for the dummy warmup run. Returns 0 to skip warmup."""
|
|
|
|
model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name)
|
|
if model_cls is not None and hasattr(model_cls, "dummy_run_num_frames"):
|
|
return int(getattr(model_cls, "dummy_run_num_frames"))
|
|
return 2 if supports_audio_input or supports_audio_output(model_class_name) else 1
|
|
|
|
|
|
def get_dlo_dummy_run_recipe(model_class_name: str) -> dict[str, Any] | None:
|
|
"""Get the model-declared DLO startup-warmup recipe, or None.
|
|
|
|
Models that opt out of the generic dummy run (``dummy_run_num_frames = 0``)
|
|
because their request geometry needs model-specific sampling arguments can
|
|
still declare a recipe used only when distributed layerwise offload is
|
|
enabled, where one startup generation primes the component allocator-cache
|
|
retention policy.
|
|
"""
|
|
|
|
model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name)
|
|
recipe = getattr(model_cls, "dlo_dummy_run_recipe", None) if model_cls is not None else None
|
|
return dict(recipe) if recipe else None
|
|
|
|
|
|
def get_dummy_run_num_image_inputs(model_class_name: str) -> int:
|
|
"""Return the maximum advertised image-input count for profiling."""
|
|
|
|
return get_diffusion_model_metadata(model_class_name).max_multimodal_image_inputs or 1
|