[Diffusion] Add Minimax-H3 modular pipeline support (#5720)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: lishunyang12 <lishunyang12@163.com> Signed-off-by: David Chen <530634352@qq.com> Co-authored-by: lishunyang12 <lishunyang12@163.com> Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com> Co-authored-by: David Chen <530634352@qq.com>
This commit is contained in:
25 files changed
+843
-177
No files matched your search
+73
-32
@@ -3,10 +3,13 @@
|
||||
|
||||
"""Generic Cache-DiT backend lifecycle and integration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any, TypeAlias
|
||||
from typing import TYPE_CHECKING, Any, TypeAlias, cast
|
||||
|
||||
import cache_dit
|
||||
import torch.nn as nn
|
||||
from cache_dit import BlockAdapter, DBCacheConfig
|
||||
from cache_dit.caching.cache_adapters.cache_adapter import CachedAdapter
|
||||
from vllm.logger import init_logger
|
||||
@@ -18,6 +21,9 @@ from vllm_omni.diffusion.cache.cachedit.config import (
|
||||
)
|
||||
from vllm_omni.diffusion.data import DiffusionCacheConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
RefreshCacheContextFunc: TypeAlias = Callable[[Any, int, bool], None]
|
||||
@@ -27,14 +33,18 @@ CacheDiTEnabler: TypeAlias = Callable[[Any, DiffusionCacheConfig], RefreshCacheC
|
||||
CUSTOM_DIT_ENABLERS: dict[str, CacheDiTEnabler] = {}
|
||||
|
||||
|
||||
def cache_summary(pipeline: Any, details: bool = True) -> None:
|
||||
def _dit_module_names(pipeline: SupportsComponentDiscovery) -> tuple[str, ...]:
|
||||
"""Return the pipeline DiT attributes that are present at runtime."""
|
||||
names = getattr(pipeline, "_dit_modules", None)
|
||||
if not isinstance(names, (list, tuple)):
|
||||
names = ("transformer",)
|
||||
return tuple(name for name in names if isinstance(name, str) and getattr(pipeline, name, None) is not None)
|
||||
|
||||
|
||||
def cache_summary(pipeline: SupportsComponentDiscovery, details: bool = True) -> None:
|
||||
"""Log Cache-DiT statistics for every transformer on the pipeline."""
|
||||
|
||||
transformers = [
|
||||
transformer
|
||||
for attribute in ("transformer", "transformer_2")
|
||||
if (transformer := getattr(pipeline, attribute, None)) is not None
|
||||
]
|
||||
transformers = [getattr(pipeline, name) for name in _dit_module_names(pipeline)]
|
||||
for transformer in transformers:
|
||||
cache_dit.summary(transformer, details=details)
|
||||
|
||||
@@ -42,19 +52,34 @@ def cache_summary(pipeline: Any, details: bool = True) -> None:
|
||||
logger.warning("CacheDiT summary failed; this pipeline has no defined transformer attribute")
|
||||
|
||||
|
||||
def _default_get_pipeline_transformer(pipeline: Any) -> Any:
|
||||
return pipeline.transformer
|
||||
def _default_get_pipeline_transformer(
|
||||
pipeline: SupportsComponentDiscovery,
|
||||
) -> nn.Module:
|
||||
return cast(nn.Module, getattr(pipeline, "transformer"))
|
||||
|
||||
|
||||
def _make_pipeline_transformer_getter(
|
||||
name: str,
|
||||
) -> Callable[[SupportsComponentDiscovery], nn.Module]:
|
||||
def get_pipeline_transformer(
|
||||
pipeline: SupportsComponentDiscovery,
|
||||
) -> nn.Module:
|
||||
return cast(nn.Module, getattr(pipeline, name))
|
||||
|
||||
return get_pipeline_transformer
|
||||
|
||||
|
||||
def _build_cache_context_refresh(
|
||||
cache_config: DiffusionCacheConfig,
|
||||
get_pipeline_transformer: Callable[[Any], Any] = _default_get_pipeline_transformer,
|
||||
get_pipeline_transformer: Callable[[SupportsComponentDiscovery], nn.Module] = _default_get_pipeline_transformer,
|
||||
) -> RefreshCacheContextFunc:
|
||||
"""Build the cache context refresh callback for one transformer."""
|
||||
|
||||
projected_config = CacheDiTConfig.from_diffusion_config(cache_config)
|
||||
|
||||
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
def refresh_cache_context(
|
||||
pipeline: SupportsComponentDiscovery, num_inference_steps: int, verbose: bool = True
|
||||
) -> None:
|
||||
transformer = get_pipeline_transformer(pipeline)
|
||||
|
||||
# Cache-DiT has no predefined SCM mask for these small step counts.
|
||||
@@ -80,10 +105,11 @@ def _build_cache_context_refresh(
|
||||
|
||||
|
||||
def enable_cache_for_dit(
|
||||
pipeline: Any,
|
||||
pipeline: SupportsComponentDiscovery,
|
||||
cache_config: DiffusionCacheConfig,
|
||||
block_adapter: BlockAdapter | None = None,
|
||||
adapter_cls: type[CachedAdapter] | None = None,
|
||||
get_pipeline_transformer: Callable[[SupportsComponentDiscovery], nn.Module] = _default_get_pipeline_transformer,
|
||||
) -> RefreshCacheContextFunc:
|
||||
"""Enable Cache-DiT for a standard single-transformer DiT pipeline."""
|
||||
|
||||
@@ -98,7 +124,7 @@ def enable_cache_for_dit(
|
||||
db_cache_config.max_warmup_steps,
|
||||
)
|
||||
|
||||
transformer = _default_get_pipeline_transformer(pipeline)
|
||||
transformer = get_pipeline_transformer(pipeline)
|
||||
cache_target = transformer if block_adapter is None else block_adapter
|
||||
if adapter_cls is not None:
|
||||
adapter_cls.apply(
|
||||
@@ -128,13 +154,16 @@ def enable_cache_for_dit(
|
||||
calibrator_config=calibrator_config,
|
||||
)
|
||||
|
||||
return _build_cache_context_refresh(cache_config)
|
||||
return _build_cache_context_refresh(cache_config, get_pipeline_transformer)
|
||||
|
||||
|
||||
def _maybe_build_block_adapter(pipeline: Any) -> BlockAdapter | None:
|
||||
def _maybe_build_block_adapter(
|
||||
pipeline: SupportsComponentDiscovery,
|
||||
get_pipeline_transformer: Callable[[SupportsComponentDiscovery], nn.Module] = _default_get_pipeline_transformer,
|
||||
) -> BlockAdapter | None:
|
||||
"""Build the model-declared block adapter, when one is configured."""
|
||||
|
||||
transformer = _default_get_pipeline_transformer(pipeline)
|
||||
transformer = get_pipeline_transformer(pipeline)
|
||||
adapter_config: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None)
|
||||
if adapter_config is None:
|
||||
logger.info(
|
||||
@@ -160,10 +189,13 @@ def _maybe_build_block_adapter(pipeline: Any) -> BlockAdapter | None:
|
||||
)
|
||||
|
||||
|
||||
def _maybe_get_cached_adapter_cls(pipeline: Any) -> type[CachedAdapter] | None:
|
||||
def _maybe_get_cached_adapter_cls(
|
||||
pipeline: SupportsComponentDiscovery,
|
||||
get_pipeline_transformer: Callable[[SupportsComponentDiscovery], nn.Module] = _default_get_pipeline_transformer,
|
||||
) -> type[CachedAdapter] | None:
|
||||
"""Return the custom cached adapter declared by the transformer."""
|
||||
|
||||
transformer = _default_get_pipeline_transformer(pipeline)
|
||||
transformer = get_pipeline_transformer(pipeline)
|
||||
adapter_config: CacheDiTAdapterConfig | None = getattr(transformer, "_cache_dit_adapter_config", None)
|
||||
return None if adapter_config is None else adapter_config.cached_adapter_cls
|
||||
|
||||
@@ -180,29 +212,37 @@ class CacheDiTBackend(CacheBackend):
|
||||
config = cache_config
|
||||
|
||||
super().__init__(config)
|
||||
self._refresh_func: RefreshCacheContextFunc | None = None
|
||||
self._refresh_funcs: list[RefreshCacheContextFunc] = []
|
||||
|
||||
def enable(self, pipeline: Any) -> None:
|
||||
def enable(self, pipeline: SupportsComponentDiscovery) -> None:
|
||||
pipeline_name = type(pipeline).__name__
|
||||
custom_enabler = CUSTOM_DIT_ENABLERS.get(pipeline_name)
|
||||
if custom_enabler is not None:
|
||||
logger.info("Using custom cache-dit enabler for model: %s", pipeline_name)
|
||||
self._refresh_func = custom_enabler(pipeline, self.config)
|
||||
self._refresh_funcs = [custom_enabler(pipeline, self.config)]
|
||||
else:
|
||||
block_adapter = _maybe_build_block_adapter(pipeline)
|
||||
adapter_cls = _maybe_get_cached_adapter_cls(pipeline)
|
||||
self._refresh_func = enable_cache_for_dit(
|
||||
pipeline,
|
||||
self.config,
|
||||
block_adapter,
|
||||
adapter_cls,
|
||||
)
|
||||
self._refresh_funcs = []
|
||||
for name in _dit_module_names(pipeline):
|
||||
get_transformer = _make_pipeline_transformer_getter(name)
|
||||
block_adapter = _maybe_build_block_adapter(pipeline, get_transformer)
|
||||
adapter_cls = _maybe_get_cached_adapter_cls(pipeline, get_transformer)
|
||||
self._refresh_funcs.append(
|
||||
enable_cache_for_dit(
|
||||
pipeline,
|
||||
self.config,
|
||||
block_adapter,
|
||||
adapter_cls,
|
||||
get_transformer,
|
||||
)
|
||||
)
|
||||
if not self._refresh_funcs:
|
||||
raise ValueError(f"Pipeline {pipeline_name} has no declared DiT modules for Cache-DiT")
|
||||
|
||||
self.enabled = True
|
||||
logger.info("Cache-dit enabled successfully on %s", pipeline_name)
|
||||
|
||||
def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
if not self.enabled or self._refresh_func is None:
|
||||
def refresh(self, pipeline: SupportsComponentDiscovery, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
if not self.enabled or not self._refresh_funcs:
|
||||
logger.warning("Cache-dit is not enabled. Cannot refresh cache context.")
|
||||
return
|
||||
|
||||
@@ -211,7 +251,8 @@ class CacheDiTBackend(CacheBackend):
|
||||
"Refreshing cache context for transformer with num_inference_steps: %s",
|
||||
num_inference_steps,
|
||||
)
|
||||
self._refresh_func(pipeline, num_inference_steps, verbose)
|
||||
for refresh_func in self._refresh_funcs:
|
||||
refresh_func(pipeline, num_inference_steps, verbose)
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
||||
+20
-12
@@ -541,16 +541,15 @@ def resolve_model_class_name(model: str | None, diffusion_load_format: str = "de
|
||||
"""
|
||||
from vllm.transformers_utils.config import get_hf_file_to_dict
|
||||
|
||||
from vllm_omni.diffusion.utils.hf_utils import get_diffusion_model_index
|
||||
|
||||
if not model:
|
||||
return None
|
||||
|
||||
is_lance_subfolder = os.path.basename(str(model).rstrip("/")) in {"Lance_3B", "Lance_3B_Video"}
|
||||
|
||||
# Diffusers models: read _class_name from model_index.json.
|
||||
try:
|
||||
model_index = get_hf_file_to_dict("model_index.json", model)
|
||||
except Exception:
|
||||
model_index = None
|
||||
# Diffusers models: read _class_name from the pipeline index.
|
||||
model_index = get_diffusion_model_index(model)
|
||||
if model_index is not None:
|
||||
return model_index.get("_class_name")
|
||||
if diffusion_load_format == "diffusers":
|
||||
@@ -603,6 +602,10 @@ class OmniDiffusionConfig:
|
||||
|
||||
model_class_name: str | None = None
|
||||
|
||||
# Optional model-defined startup task. Pipelines may use this to select
|
||||
# task-specific components or weights before serving requests.
|
||||
task_type: str | None = None
|
||||
|
||||
dtype: torch.dtype = torch.bfloat16
|
||||
|
||||
model_config: dict[str, Any] = field(default_factory=dict)
|
||||
@@ -1143,18 +1146,23 @@ class OmniDiffusionConfig:
|
||||
def enrich_config(self) -> None:
|
||||
"""Load model metadata from HuggingFace and populate config fields.
|
||||
|
||||
Diffusers-style models expose ``model_index.json`` with ``_class_name``.
|
||||
Diffusers-style models expose a pipeline index with ``_class_name``.
|
||||
Non-diffusers models (e.g. Bagel, NextStep) only have ``config.json``,
|
||||
so we fall back to reading that and mapping model_type manually.
|
||||
"""
|
||||
from vllm.transformers_utils.config import get_hf_file_to_dict
|
||||
|
||||
from vllm_omni.diffusion.utils.hf_utils import get_diffusion_model_index
|
||||
|
||||
# Default model_class_name for diffusers adapter
|
||||
if self.model_class_name is None and self.diffusion_load_format == "diffusers":
|
||||
self.model_class_name = "DiffusersAdapterPipeline"
|
||||
|
||||
try:
|
||||
config_dict = get_hf_file_to_dict("model_index.json", self.model)
|
||||
config_dict = get_diffusion_model_index(
|
||||
self.model,
|
||||
revision=self.revision,
|
||||
)
|
||||
if config_dict is not None:
|
||||
if self.model_class_name is None:
|
||||
self.model_class_name = config_dict.get("_class_name", None)
|
||||
@@ -1169,7 +1177,7 @@ class OmniDiffusionConfig:
|
||||
self.diffusers_pipeline_cls = getattr(diffusers, diffusers_pipeline_cls_name)
|
||||
except (KeyError, AttributeError) as exc:
|
||||
logger.warning(
|
||||
"Could not find valid _class_name for diffusers pipeline in model_index.json: %s. "
|
||||
"Could not find a valid _class_name in the Diffusers pipeline index: %s. "
|
||||
"Without the underlying pipeline class the dummy run may omit required inputs.",
|
||||
exc,
|
||||
)
|
||||
@@ -1182,16 +1190,16 @@ class OmniDiffusionConfig:
|
||||
else:
|
||||
self.set_tf_model_config(TransformerConfig())
|
||||
else:
|
||||
raise FileNotFoundError("model_index.json not found")
|
||||
raise FileNotFoundError("Diffusers pipeline index not found")
|
||||
except (AttributeError, OSError, ValueError, FileNotFoundError):
|
||||
# Skip transformer config loading for diffusers adapter
|
||||
# (non-DiT models don't have a separate transformer folder/config)
|
||||
if self.diffusion_load_format == "diffusers":
|
||||
self.set_tf_model_config(TransformerConfig())
|
||||
logger.warning(
|
||||
"Could not find valid model_index.json per diffusers format. "
|
||||
"Could not find a valid pipeline index per Diffusers format. "
|
||||
"This model is likely unsupported by the diffusers backend. "
|
||||
"Also, without knowing the underlying diffusers pipeline class from model_index.json, "
|
||||
"Also, without knowing the underlying pipeline class from its index, "
|
||||
"the dummy run will input only text prompt, which may cause errors for pipelines "
|
||||
"that require additional inputs."
|
||||
)
|
||||
@@ -1207,7 +1215,7 @@ class OmniDiffusionConfig:
|
||||
self.set_tf_model_config(TransformerConfig())
|
||||
self.update_multimodal_support()
|
||||
return
|
||||
raise ValueError(f"Could not find config.json or model_index.json for model {self.model}")
|
||||
raise ValueError(f"Could not find config.json or a Diffusers pipeline index for {self.model}")
|
||||
|
||||
self.set_tf_model_config(TransformerConfig.from_dict(cfg))
|
||||
model_type = cfg.get("model_type")
|
||||
|
||||
@@ -366,8 +366,8 @@ class DiffusionLoRAManager:
|
||||
fully_sharded_loras=False,
|
||||
)
|
||||
|
||||
# Default denoising components plus any a pipeline opts into via
|
||||
# ``_lora_components`` (opt-in; other pipelines unchanged).
|
||||
# Default denoising components, declared DiT components, and any a
|
||||
# pipeline opts into via ``_lora_components``.
|
||||
#
|
||||
# NOTE: SDXL-style pipelines expose the denoiser as ``unet``.
|
||||
# Without scanning this component, adapters can load/activate while
|
||||
@@ -379,8 +379,10 @@ class DiffusionLoRAManager:
|
||||
"bagel",
|
||||
"unet",
|
||||
)
|
||||
declared_components = tuple(getattr(self.pipeline, "_dit_modules", ()) or ())
|
||||
extra_components = tuple(getattr(self.pipeline, "_lora_components", ()) or ())
|
||||
for component_name in (*default_components, *extra_components):
|
||||
component_names = dict.fromkeys((*default_components, *declared_components, *extra_components))
|
||||
for component_name in component_names:
|
||||
if not hasattr(self.pipeline, component_name):
|
||||
continue
|
||||
component = getattr(self.pipeline, component_name)
|
||||
|
||||
@@ -45,6 +45,13 @@ _DIFFUSION_MODEL_METADATA: dict[str, DiffusionModelMetadata] = {
|
||||
# packed TRTLLM backend consumes cu_seqlens and isolates that padding.
|
||||
attention_mask_free=True,
|
||||
),
|
||||
# The modular alias is served by MiniMaxH3Pipeline and has the same
|
||||
# Ref2VA request contract. Keep admission limits in sync with it.
|
||||
"MiniMaxH3ModularPipeline": DiffusionModelMetadata(
|
||||
supports_multimodal_inputs=True,
|
||||
max_multimodal_image_inputs=9,
|
||||
supports_mixed_reference_inputs=True,
|
||||
),
|
||||
"WanPipeline": DiffusionModelMetadata(attention_mask_free=True),
|
||||
"WanImageToVideoPipeline": DiffusionModelMetadata(attention_mask_free=True),
|
||||
"WanVACEPipeline": DiffusionModelMetadata(attention_mask_free=True),
|
||||
|
||||
@@ -9,6 +9,7 @@ import os
|
||||
import tempfile
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from contextlib import contextmanager
|
||||
from itertools import groupby
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
|
||||
@@ -44,6 +45,9 @@ from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import (
|
||||
)
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.errors import OmniClientError
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf_specific,
|
||||
)
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .condition_noise import (
|
||||
@@ -107,6 +111,57 @@ MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES = 30 * 1024 * 1024
|
||||
MINIMAX_H3_REFERENCE_IMAGE_FORMATS = frozenset({"jpeg", "png", "webp", "heic", "heif"})
|
||||
MINIMAX_H3_MIN_OUTPUT_SECONDS = 4.0
|
||||
MINIMAX_H3_MAX_OUTPUT_SECONDS = 15.0
|
||||
MINIMAX_H3_DOWNLOAD_PATTERNS = [
|
||||
"FL2VA/**",
|
||||
"Ref2VA/model_index.json",
|
||||
"Ref2VA/transformer/**",
|
||||
]
|
||||
MINIMAX_H3_TASK_DOWNLOAD_PATTERNS = {
|
||||
"fl2va": ["FL2VA/**"],
|
||||
"ref2va": ["Ref2VA/**"],
|
||||
}
|
||||
|
||||
|
||||
def _minimax_h3_partition_for_task(
|
||||
task_type: str | None,
|
||||
model: str | None = None,
|
||||
) -> str:
|
||||
task = str(task_type or "auto").lower()
|
||||
if task == "auto" and model is not None:
|
||||
path = Path(model)
|
||||
if path.is_dir() and path.name in {"FL2VA", "Ref2VA"} and (path / "model_index.json").is_file():
|
||||
return path.name.lower()
|
||||
if task in {"auto", "combined"}:
|
||||
return "combined"
|
||||
if task in {"t2va", "fl2va"}:
|
||||
return "fl2va"
|
||||
if task == "ref2va":
|
||||
return "ref2va"
|
||||
raise ValueError(f"MiniMax-H3 task_type must be one of auto, t2va, fl2va, or ref2va; got {task_type!r}")
|
||||
|
||||
|
||||
def _resolve_minimax_h3_model_root(
|
||||
model: str,
|
||||
revision: str | None,
|
||||
partition: str,
|
||||
) -> Path:
|
||||
path = Path(model)
|
||||
if path.is_dir():
|
||||
if path.name in {"FL2VA", "Ref2VA"} and (path / "model_index.json").is_file():
|
||||
return path.parent
|
||||
return path
|
||||
allow_patterns = (
|
||||
MINIMAX_H3_DOWNLOAD_PATTERNS if partition == "combined" else MINIMAX_H3_TASK_DOWNLOAD_PATTERNS[partition]
|
||||
)
|
||||
return Path(
|
||||
download_weights_from_hf_specific(
|
||||
model_name_or_path=model,
|
||||
cache_dir=None,
|
||||
allow_patterns=allow_patterns,
|
||||
revision=revision,
|
||||
require_all=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _resolve_component_quant_config(quant_config, component: str):
|
||||
@@ -457,11 +512,7 @@ class MiniMaxH3Pipeline(
|
||||
):
|
||||
"""CFG-distilled joint video/audio generation for MiniMax H3."""
|
||||
|
||||
_dit_modules: ClassVar[list[str]] = ["transformer"]
|
||||
# TODO(offload): Reconcile this static plan with the modular H3 pipeline
|
||||
# (#5720). Its combined mode has ``transformer`` and ``transformers_ref``;
|
||||
# build the resident/staging plan from the selected task partition so a
|
||||
# consumer-GPU single-task server never stages both DiTs.
|
||||
_dit_modules: ClassVar[list[str]] = ["transformer", "transformers_ref"]
|
||||
_encoder_modules: ClassVar[list[str]] = ["text_encoder"]
|
||||
_vae_modules: ClassVar[list[str]] = ["video_vae", "audio_vae"]
|
||||
_offload_plan: ClassVar[OffloadPlan] = OffloadPlan(
|
||||
@@ -500,24 +551,64 @@ class MiniMaxH3Pipeline(
|
||||
if int(self.parallel_config.cfg_parallel_size) != 1:
|
||||
raise ValueError("MiniMax-H3 is CFG-distilled and has no negative branch; cfg_parallel_size must be 1")
|
||||
self.device = get_local_device()
|
||||
model_path = str(od_config.model)
|
||||
model_index = json.loads((Path(model_path) / "model_index.json").read_text(encoding="utf-8"))
|
||||
self.partition = _minimax_h3_partition_for_task(
|
||||
getattr(od_config, "task_type", None),
|
||||
str(od_config.model),
|
||||
)
|
||||
model_root = _resolve_minimax_h3_model_root(
|
||||
str(od_config.model),
|
||||
od_config.revision,
|
||||
self.partition,
|
||||
)
|
||||
model_path = model_root / ("Ref2VA" if self.partition == "ref2va" else "FL2VA")
|
||||
model_index = json.loads((model_path / "model_index.json").read_text(encoding="utf-8"))
|
||||
release = model_index.get("_minimax_h3") or {}
|
||||
self.partition = str(release.get("partition", ""))
|
||||
self.supported_tasks = frozenset(release.get("tasks") or ())
|
||||
partition = str(release.get("partition", "")).lower()
|
||||
expected_partition = "ref2va" if self.partition == "ref2va" else "fl2va"
|
||||
if partition != expected_partition:
|
||||
raise ValueError(f"invalid MiniMax-H3 {expected_partition} partition at {model_path}")
|
||||
|
||||
supported_tasks = {str(task).lower() for task in release.get("tasks", [])}
|
||||
if not supported_tasks:
|
||||
supported_tasks = {"ref2va"} if partition == "ref2va" else {"t2va", "fl2va"}
|
||||
ref2va_model_path = None
|
||||
if self.partition == "combined":
|
||||
ref2va_model_path = model_root / "Ref2VA"
|
||||
ref2va_index_path = ref2va_model_path / "model_index.json"
|
||||
if not ref2va_index_path.is_file():
|
||||
raise ValueError(f"Ref2VA partition not found at {ref2va_model_path}")
|
||||
ref2va_index = json.loads(ref2va_index_path.read_text(encoding="utf-8"))
|
||||
ref2va_release = ref2va_index.get("_minimax_h3") or {}
|
||||
if str(ref2va_release.get("partition", "")).lower() != "ref2va":
|
||||
raise ValueError(f"invalid MiniMax-H3 ref2va partition at {ref2va_model_path}")
|
||||
supported_tasks.update(str(task).lower() for task in ref2va_release.get("tasks", ["ref2va"]))
|
||||
|
||||
self.supported_tasks = frozenset(supported_tasks)
|
||||
shifts = release.get("sigma_shift_scales") or {}
|
||||
self.default_video_shift = float(shifts.get("video", 12.0))
|
||||
self.default_audio_shift = float(shifts.get("audio", 3.0))
|
||||
|
||||
self.weights_sources = [
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=model_path,
|
||||
model_or_path=str(model_path),
|
||||
subfolder="transformer",
|
||||
revision=od_config.revision,
|
||||
prefix="transformer.",
|
||||
fall_back_to_pt=False,
|
||||
)
|
||||
]
|
||||
self._dit_modules = ["transformer"]
|
||||
if ref2va_model_path is not None:
|
||||
self.weights_sources.append(
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=str(ref2va_model_path),
|
||||
subfolder="transformer",
|
||||
revision=od_config.revision,
|
||||
prefix="transformers_ref.",
|
||||
fall_back_to_pt=False,
|
||||
)
|
||||
)
|
||||
self._dit_modules.append("transformers_ref")
|
||||
transformer_quant_config = _resolve_component_quant_config(
|
||||
od_config.quantization_config,
|
||||
"transformer",
|
||||
@@ -526,14 +617,19 @@ class MiniMaxH3Pipeline(
|
||||
od_config,
|
||||
quant_config=transformer_quant_config,
|
||||
)
|
||||
if ref2va_model_path is not None:
|
||||
self.transformers_ref = MiniMaxH3DiTModel(
|
||||
od_config,
|
||||
quant_config=transformer_quant_config,
|
||||
)
|
||||
|
||||
self.tokenizer = Qwen2TokenizerFast.from_pretrained(
|
||||
model_path,
|
||||
str(model_path),
|
||||
subfolder="tokenizer",
|
||||
local_files_only=os.path.isdir(model_path),
|
||||
)
|
||||
self.processor = Qwen3VLProcessor.from_pretrained(
|
||||
model_path,
|
||||
str(model_path),
|
||||
subfolder="processor",
|
||||
local_files_only=os.path.isdir(model_path),
|
||||
)
|
||||
@@ -588,16 +684,23 @@ class MiniMaxH3Pipeline(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
prefix = "transformer."
|
||||
def source_prefix(item: tuple[str, torch.Tensor]) -> str:
|
||||
name, _ = item
|
||||
prefix = name.partition(".")[0] + "."
|
||||
if prefix in {"transformer.", "transformers_ref."}:
|
||||
return prefix
|
||||
raise ValueError(f"unexpected MiniMax-H3 weight {name!r}")
|
||||
|
||||
def transformer_weights():
|
||||
for name, tensor in weights:
|
||||
if name.startswith(prefix):
|
||||
yield name[len(prefix) :], tensor
|
||||
|
||||
loaded = self.transformer.load_weights(transformer_weights())
|
||||
self.transformer.post_load_weights()
|
||||
loaded_with_prefix = {prefix + name for name in loaded}
|
||||
loaded_with_prefix: set[str] = set()
|
||||
loaded_prefixes: set[str] = set()
|
||||
for prefix, grouped_weights in groupby(weights, key=source_prefix):
|
||||
if prefix in loaded_prefixes:
|
||||
raise ValueError(f"MiniMax-H3 weight source {prefix!r} is not contiguous")
|
||||
loaded_prefixes.add(prefix)
|
||||
transformer = getattr(self, prefix.removesuffix("."))
|
||||
loaded = transformer.load_weights((name[len(prefix) :], tensor) for name, tensor in grouped_weights)
|
||||
transformer.post_load_weights()
|
||||
loaded_with_prefix.update(prefix + name for name in loaded)
|
||||
# The text encoder and both VAEs load eagerly in ``__init__`` rather
|
||||
# than through ``weights_sources``. Record them for the runner's strict
|
||||
# missing-parameter check.
|
||||
@@ -606,14 +709,23 @@ class MiniMaxH3Pipeline(
|
||||
loaded_with_prefix.update(f"{component_name}.{name}" for name, _ in component.named_parameters())
|
||||
return loaded_with_prefix
|
||||
|
||||
def _transformer_for_task(self, task: str) -> MiniMaxH3DiTModel:
|
||||
if task == "ref2va" and hasattr(self, "transformers_ref"):
|
||||
return self.transformers_ref
|
||||
return self.transformer
|
||||
|
||||
def _resolve_task(
|
||||
self,
|
||||
requested: str | None,
|
||||
multi_modal_data: dict[str, Any],
|
||||
) -> str:
|
||||
if requested is None:
|
||||
# A Ref2VA-only startup has no FL2VA transformer; preserve its
|
||||
# historical implicit default even for image-only references.
|
||||
if self.partition == "ref2va":
|
||||
requested = "ref2va"
|
||||
elif multi_modal_data.get("video") is not None or multi_modal_data.get("audio") is not None:
|
||||
requested = "ref2va"
|
||||
elif multi_modal_data.get("image") is not None:
|
||||
requested = "fl2va"
|
||||
else:
|
||||
@@ -1228,14 +1340,14 @@ class MiniMaxH3Pipeline(
|
||||
return video_rows, audio_rows
|
||||
|
||||
@contextmanager
|
||||
def _resident_dit_layers_on_device(self):
|
||||
def _resident_dit_layers_on_device(self, *, enabled: bool = True):
|
||||
controller = getattr(self, "_dlo_residency_controller", None)
|
||||
if controller is not None:
|
||||
if controller is not None and enabled:
|
||||
controller.load_resident_layers()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if controller is not None:
|
||||
if controller is not None and enabled:
|
||||
controller.offload_resident_layers()
|
||||
|
||||
def diffuse(
|
||||
@@ -1359,10 +1471,15 @@ class MiniMaxH3Pipeline(
|
||||
num_steps=num_steps,
|
||||
shift_scale=audio_shift,
|
||||
)
|
||||
with self._resident_dit_layers_on_device():
|
||||
transformer = self._transformer_for_task(task)
|
||||
# The static DLO plan keeps leading blocks resident only for the
|
||||
# primary ``transformer``. In combined mode ``transformers_ref`` is
|
||||
# fully streamed, so a Ref2VA request must not stage the inactive
|
||||
# FL2VA transformer resident blocks.
|
||||
with self._resident_dit_layers_on_device(enabled=transformer is self.transformer):
|
||||
with self.progress_bar(total=len(video_sigmas) - 1) as progress:
|
||||
video_rows, audio_rows = minimax_h3_denoise_loop(
|
||||
model=self.transformer,
|
||||
model=transformer,
|
||||
positive=branch,
|
||||
initial_video_rows=initial_video,
|
||||
initial_audio_rows=initial_audio,
|
||||
|
||||
@@ -91,6 +91,11 @@ _DIFFUSION_MODELS = {
|
||||
"pipeline_minimax_h3",
|
||||
"MiniMaxH3Pipeline",
|
||||
),
|
||||
"MiniMaxH3ModularPipeline": (
|
||||
"minimax_h3",
|
||||
"pipeline_minimax_h3",
|
||||
"MiniMaxH3Pipeline",
|
||||
),
|
||||
"StableAudioPipeline": (
|
||||
"stable_audio",
|
||||
"pipeline_stable_audio",
|
||||
@@ -427,9 +432,11 @@ def _apply_sequence_parallel_if_enabled(model, od_config: OmniDiffusionConfig) -
|
||||
if sp_size <= 1:
|
||||
return
|
||||
|
||||
# Find transformer model(s) in the pipeline that have _sp_plan
|
||||
# Include transformer_2 for two-stage models (e.g., Wan MoE)
|
||||
transformer_attrs = ["transformer", "transformer_2", "dit", "unet"]
|
||||
# Prefer the pipeline's declared DiT components so custom component
|
||||
# names receive the same SP hooks as conventional transformer names.
|
||||
transformer_attrs = getattr(model, "_dit_modules", None)
|
||||
if not transformer_attrs:
|
||||
transformer_attrs = ("transformer", "transformer_2", "dit", "unet")
|
||||
applied_count = 0
|
||||
|
||||
for attr in transformer_attrs:
|
||||
@@ -508,6 +515,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = {
|
||||
"LTX2T2VDMD2Pipeline": "get_ltx2_post_process_func",
|
||||
"LTX2I2VDMD2Pipeline": "get_ltx2_post_process_func",
|
||||
"MiniMaxH3Pipeline": "get_minimax_h3_post_process_func",
|
||||
"MiniMaxH3ModularPipeline": "get_minimax_h3_post_process_func",
|
||||
"StableAudioPipeline": "get_stable_audio_post_process_func",
|
||||
"SoulXSingerPipeline": "get_soulxsinger_post_process_func",
|
||||
"SoulXSingerSVCPipeline": "get_soulxsinger_post_process_func",
|
||||
|
||||
@@ -7,6 +7,24 @@ from vllm.transformers_utils.config import get_hf_file_to_dict
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
DIFFUSION_MODEL_INDEX_FILES = (
|
||||
"model_index.json",
|
||||
"modular_model_index.json",
|
||||
)
|
||||
|
||||
|
||||
def get_diffusion_model_index(
|
||||
model_name: str,
|
||||
*,
|
||||
revision: str | None = None,
|
||||
) -> dict | None:
|
||||
"""Read the first standard Diffusers pipeline index available."""
|
||||
for filename in DIFFUSION_MODEL_INDEX_FILES:
|
||||
config = get_hf_file_to_dict(filename, model_name, revision=revision)
|
||||
if isinstance(config, Mapping):
|
||||
return dict(config)
|
||||
return None
|
||||
|
||||
|
||||
def load_diffusers_config(model_name) -> dict:
|
||||
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
||||
@@ -58,33 +76,32 @@ def is_diffusion_model(model_name: str) -> bool:
|
||||
"""Check if a model is a diffusion model.
|
||||
|
||||
Uses multiple fallback strategies to detect diffusion models:
|
||||
1. Check local file system for model_index.json (fastest, no imports)
|
||||
1. Check the local file system for a standard Diffusers index
|
||||
2. Check using vllm's get_hf_file_to_dict utility
|
||||
3. Try the standard diffusers approach (may fail due to import issues)
|
||||
"""
|
||||
# Strategy 1: Check local file system first (fastest, avoids import issues)
|
||||
if os.path.isdir(model_name):
|
||||
model_index_path = os.path.join(model_name, "model_index.json")
|
||||
if os.path.exists(model_index_path):
|
||||
for filename in DIFFUSION_MODEL_INDEX_FILES:
|
||||
model_index_path = os.path.join(model_name, filename)
|
||||
if not os.path.exists(model_index_path):
|
||||
continue
|
||||
try:
|
||||
import json
|
||||
|
||||
with open(model_index_path) as f:
|
||||
config_dict = json.load(f)
|
||||
if config_dict.get("_class_name") and config_dict.get("_diffusers_version"):
|
||||
logger.debug("Detected diffusion model via local model_index.json")
|
||||
logger.debug("Detected diffusion model via local %s", filename)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug("Failed to read local model_index.json: %s", e)
|
||||
logger.debug("Failed to read local %s: %s", filename, e)
|
||||
|
||||
# Strategy 2: Check using vllm's utility (works for both local and remote models)
|
||||
try:
|
||||
config_dict = get_hf_file_to_dict("model_index.json", model_name)
|
||||
if config_dict is not None and config_dict.get("_class_name") and config_dict.get("_diffusers_version"):
|
||||
logger.debug("Detected diffusion model via model_index.json")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug("Failed to check model_index.json via get_hf_file_to_dict: %s", e)
|
||||
config_dict = get_diffusion_model_index(model_name)
|
||||
if config_dict is not None and config_dict.get("_class_name") and config_dict.get("_diffusers_version"):
|
||||
logger.debug("Detected diffusion model via a standard Diffusers index")
|
||||
return True
|
||||
|
||||
# Strategy 3: Try the standard diffusers approach (may fail due to import issues)
|
||||
# This is last because it requires importing diffusers/xformers/flash_attn
|
||||
|
||||
@@ -280,8 +280,11 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
exc,
|
||||
)
|
||||
else:
|
||||
self._compile_transformer("transformer")
|
||||
self._compile_transformer("transformer_2")
|
||||
transformer_attrs = getattr(self.pipeline, "_dit_modules", None)
|
||||
if not transformer_attrs:
|
||||
transformer_attrs = ("transformer", "transformer_2")
|
||||
for attr_name in transformer_attrs:
|
||||
self._compile_transformer(attr_name)
|
||||
else:
|
||||
logger.warning(
|
||||
"Model runner: Platform %s does not support torch inductor, skipping torch.compile.",
|
||||
|
||||
Reference in new issue
Block a user