[FEATURE] Tile/Patch parallelism refactor for easily support other models (#1366)

Signed-off-by: dengyunyang <584797741@qq.com>
This commit is contained in:
dengyunyang authored and GitHub committed 2026-03-03 15:48:51 +08:00
1 parent e7f2c88701
commit d275c0c993
18 files changed
+1336 -60

No files matched your search

+8 -29
View File
@@ -8,6 +8,7 @@ from vllm.logger import init_logger
from vllm.model_executor.models.registry import _LazyRegisteredModel, _ModelRegistry
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import DistributedVaeMixin
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelConfig, get_sp_plan_from_model
from vllm_omni.diffusion.hooks.sequence_parallel import apply_sequence_parallel
@@ -123,13 +124,6 @@ DiffusionModelRegistry = _ModelRegistry(
}
)
_VAE_PATCH_PARALLEL_ALLOWLIST = {
# Only enable for models we have validated end-to-end.
"StableDiffusion3Pipeline",
"ZImagePipeline",
"NextStep11Pipeline",
}
_NO_CACHE_ACCELERATION = {
# Pipelines that do not support cache acceleration (cache_dit / tea_cache).
"NextStep11Pipeline",
@@ -161,17 +155,14 @@ def initialize_model(
model = model_class(od_config=od_config)
vae_pp_size = od_config.parallel_config.vae_patch_parallel_size
if vae_pp_size > 1 and od_config.model_class_name not in _VAE_PATCH_PARALLEL_ALLOWLIST:
is_distributed_vae = hasattr(model, "vae") and isinstance(model.vae, DistributedVaeMixin)
if vae_pp_size > 1 and not is_distributed_vae:
logger.warning(
"vae_patch_parallel_size=%d is set but VAE patch parallelism is only enabled for %s; ignoring.",
"vae_patch_parallel_size=%d is set but VAE patch parallelism is NOT enabled for %s; ignoring.",
vae_pp_size,
sorted(_VAE_PATCH_PARALLEL_ALLOWLIST),
od_config.model_class_name,
)
if (
vae_pp_size > 1
and od_config.model_class_name in _VAE_PATCH_PARALLEL_ALLOWLIST
and not od_config.vae_use_tiling
):
if vae_pp_size > 1 and is_distributed_vae and not od_config.vae_use_tiling:
logger.info(
"vae_patch_parallel_size=%d requires vae_use_tiling; automatically enabling it.",
vae_pp_size,
@@ -184,20 +175,8 @@ def initialize_model(
if hasattr(model.vae, "use_tiling"):
model.vae.use_tiling = od_config.vae_use_tiling
if (
vae_pp_size > 1
and hasattr(model, "vae")
and od_config.model_class_name in _VAE_PATCH_PARALLEL_ALLOWLIST
and od_config.vae_use_tiling
):
from vllm_omni.diffusion.distributed.parallel_state import get_dit_group
from vllm_omni.diffusion.distributed.vae_patch_parallel import maybe_wrap_vae_decode_with_patch_parallelism
maybe_wrap_vae_decode_with_patch_parallelism(
model,
vae_patch_parallel_size=vae_pp_size,
group_getter=get_dit_group,
)
if is_distributed_vae:
model.vae.set_parallel_size(vae_pp_size)
# Apply sequence parallelism if enabled
# This follows diffusers' pattern where enable_parallelism() is called