Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0527aadf6b | ||
|
|
6b95de36a6 |
No files matched your search
@@ -12,7 +12,7 @@ implementations to reduce numerical drift and sampling divergence.
|
|||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install kernels==0.14.1
|
pip install kernels==0.15.2
|
||||||
```
|
```
|
||||||
|
|
||||||
If `kernels` is unavailable, an explicit `FLASH_ATTN_HUB` or
|
If `kernels` is unavailable, an explicit `FLASH_ATTN_HUB` or
|
||||||
|
|||||||
@@ -120,18 +120,10 @@ Additional requirements:
|
|||||||
| Parameter | Type | Default | Description |
|
| Parameter | Type | Default | Description |
|
||||||
|-----------|------|---------|-------------|
|
|-----------|------|---------|-------------|
|
||||||
| `vae_use_tiling` | bool | False | Must be set to `True` when using VAE patch parallelism. |
|
| `vae_use_tiling` | bool | False | Must be set to `True` when using VAE patch parallelism. |
|
||||||
| `vae_stack_tiling` | `"auto"` \| `"true"` \| `"false"` | `"false"` | For VAEs with native support (currently MiniMax H3), batch local decoder tiles into fewer model calls. `"auto"` enables it when the request has at least two tiles per VAE-parallel rank. |
|
|
||||||
|
|
||||||
!!! note "Automatic VAE Tiling"
|
!!! note "Automatic VAE Tiling"
|
||||||
When `vae_patch_parallel_size > 1` and the model has a distributed VAE (`DistributedVaeMixin`), the system automatically sets `vae_use_tiling=True` if not already enabled.
|
When `vae_patch_parallel_size > 1` and the model has a distributed VAE (`DistributedVaeMixin`), the system automatically sets `vae_use_tiling=True` if not already enabled.
|
||||||
|
|
||||||
!!! tip "MiniMax H3 stacked tiling"
|
|
||||||
MiniMax H3 supports lossless native stacked tiling. Enable it with
|
|
||||||
`--vae-stack-tiling auto` (or `true`) to batch each rank's spatial tiles
|
|
||||||
into one decoder call per temporal window. `auto` keeps the sequential
|
|
||||||
path for shapes that do not provide multiple tiles per rank, while
|
|
||||||
`false` remains the default to avoid increasing the VAE activation peak.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Spatially-Sharded Decode (Wan)
|
## Spatially-Sharded Decode (Wan)
|
||||||
|
|||||||
@@ -13,8 +13,9 @@ transformers >= 5.10.1, < 5.15
|
|||||||
av>=14.0.0
|
av>=14.0.0
|
||||||
omegaconf>=2.3.0
|
omegaconf>=2.3.0
|
||||||
diffusers==0.40.0
|
diffusers==0.40.0
|
||||||
# Hub kernels, including LTX-2.5 NATTEN
|
# Hub kernels, including LTX-2.5 NATTEN. Keep this pin in sync with the
|
||||||
kernels==0.14.1
|
# supported transformers range above.
|
||||||
|
kernels==0.15.2
|
||||||
safetensors>=0.8.0
|
safetensors>=0.8.0
|
||||||
accelerate==1.12.0
|
accelerate==1.12.0
|
||||||
soundfile>=0.13.1
|
soundfile>=0.13.1
|
||||||
|
|||||||
@@ -614,54 +614,6 @@ def test_initialize_model_sets_current_diffusion_config_during_model_constructio
|
|||||||
assert get_current_diffusion_config_or_none() is None
|
assert get_current_diffusion_config_or_none() is None
|
||||||
|
|
||||||
|
|
||||||
def test_configure_vae_stack_tiling_enables_tiling_and_native_capability():
|
|
||||||
from vllm_omni.diffusion.registry import _configure_vae_stack_tiling
|
|
||||||
|
|
||||||
calls = []
|
|
||||||
vae = SimpleNamespace(
|
|
||||||
supports_stack_tiling=True,
|
|
||||||
set_stack_tiling=lambda mode: calls.append(mode),
|
|
||||||
)
|
|
||||||
model = SimpleNamespace(vae=vae)
|
|
||||||
od_config = SimpleNamespace(
|
|
||||||
model_class_name="MiniMaxH3Pipeline",
|
|
||||||
vae_stack_tiling="auto",
|
|
||||||
vae_use_tiling=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
_configure_vae_stack_tiling(model, od_config)
|
|
||||||
|
|
||||||
assert od_config.vae_use_tiling is True
|
|
||||||
assert calls == ["auto"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_configure_vae_stack_tiling_rejects_unsupported_vae():
|
|
||||||
from vllm_omni.diffusion.registry import _configure_vae_stack_tiling
|
|
||||||
|
|
||||||
od_config = SimpleNamespace(
|
|
||||||
model_class_name="UnsupportedPipeline",
|
|
||||||
vae_stack_tiling="true",
|
|
||||||
vae_use_tiling=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="does not declare stacked-tiling support"):
|
|
||||||
_configure_vae_stack_tiling(SimpleNamespace(vae=SimpleNamespace()), od_config)
|
|
||||||
|
|
||||||
|
|
||||||
def test_configure_vae_stack_tiling_auto_falls_back_for_unsupported_vae():
|
|
||||||
from vllm_omni.diffusion.registry import _configure_vae_stack_tiling
|
|
||||||
|
|
||||||
od_config = SimpleNamespace(
|
|
||||||
model_class_name="UnsupportedPipeline",
|
|
||||||
vae_stack_tiling="auto",
|
|
||||||
vae_use_tiling=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
_configure_vae_stack_tiling(SimpleNamespace(vae=SimpleNamespace()), od_config)
|
|
||||||
|
|
||||||
assert od_config.vae_use_tiling is False
|
|
||||||
|
|
||||||
|
|
||||||
def test_load_model_custom_pipeline_sets_current_diffusion_config(monkeypatch):
|
def test_load_model_custom_pipeline_sets_current_diffusion_config(monkeypatch):
|
||||||
import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod
|
import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod
|
||||||
|
|
||||||
|
|||||||
@@ -386,7 +386,7 @@ def test_ltx25_natten_failure_has_actionable_omni_remedy(monkeypatch):
|
|||||||
MissingNattenProcessor,
|
MissingNattenProcessor,
|
||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match=r"kernels==0\.14\.1.*supported GPU.*allow Hub access"):
|
with pytest.raises(RuntimeError, match=r"kernels==0\.15\.2.*supported GPU.*allow Hub access"):
|
||||||
ltx2_components._create_ltx25_natten_processor()
|
ltx2_components._create_ltx25_natten_processor()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -30,7 +30,6 @@ class _FakeCheckpointModel:
|
|||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.vae_ratio = RATIO
|
self.vae_ratio = RATIO
|
||||||
self.parallel_tiling = True
|
self.parallel_tiling = True
|
||||||
self.stack_tiling = False
|
|
||||||
self.decoded_with = []
|
self.decoded_with = []
|
||||||
|
|
||||||
def split_tiles(self, input_len, is_decoder=False):
|
def split_tiles(self, input_len, is_decoder=False):
|
||||||
@@ -58,7 +57,6 @@ def _vae(parallel_size, state):
|
|||||||
vae.model = _FakeCheckpointModel()
|
vae.model = _FakeCheckpointModel()
|
||||||
vae.remote = remote
|
vae.remote = remote
|
||||||
vae.parallel_size = parallel_size
|
vae.parallel_size = parallel_size
|
||||||
vae.stack_tiling_mode = "false"
|
|
||||||
return vae
|
return vae
|
||||||
|
|
||||||
|
|
||||||
@@ -91,33 +89,6 @@ def test_tile_count_uses_the_decoder_grid_not_the_encoder_one(h, w, decoder_tile
|
|||||||
assert as_encoder == encoder_tiles != decoder_tiles
|
assert as_encoder == encoder_tiles != decoder_tiles
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("parallel_size", "h", "w", "expected"),
|
|
||||||
[
|
|
||||||
(1, 48, 84, True),
|
|
||||||
(2, 48, 84, True),
|
|
||||||
(4, 48, 84, True),
|
|
||||||
(8, 48, 84, True),
|
|
||||||
(16, 48, 84, False),
|
|
||||||
(4, 16, 16, False),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_auto_stack_tiling_requires_multiple_tiles_per_rank(parallel_size, h, w, expected):
|
|
||||||
vae = _vae(parallel_size, {})
|
|
||||||
vae.set_stack_tiling("auto")
|
|
||||||
|
|
||||||
assert MiniMaxH3VideoVAE._should_stack_decode_tiles(vae, _latent(h, w)) is expected
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(("mode", "expected"), [(False, False), (True, True), ("false", False), ("true", True)])
|
|
||||||
def test_stack_tiling_configuration_accepts_bool_and_string_modes(mode, expected):
|
|
||||||
vae = _vae(1, {})
|
|
||||||
|
|
||||||
vae.set_stack_tiling(mode)
|
|
||||||
|
|
||||||
assert vae.model.stack_tiling is expected
|
|
||||||
|
|
||||||
|
|
||||||
def test_rank_local_tiling_restores_the_group_state():
|
def test_rank_local_tiling_restores_the_group_state():
|
||||||
state = {"sp_size": 4, "sp_rank": 2, "sp_enabled": True, "sp_process_group": "pg"}
|
state = {"sp_size": 4, "sp_rank": 2, "sp_enabled": True, "sp_process_group": "pg"}
|
||||||
vae = _vae(4, state)
|
vae = _vae(4, state)
|
||||||
@@ -185,7 +156,6 @@ def _dispatch_vae(parallel_size):
|
|||||||
seen["sp_enabled"] = state["sp_enabled"]
|
seen["sp_enabled"] = state["sp_enabled"]
|
||||||
seen["sp_process_group"] = state["sp_process_group"]
|
seen["sp_process_group"] = state["sp_process_group"]
|
||||||
seen["parallel_tiling"] = vae.model.parallel_tiling
|
seen["parallel_tiling"] = vae.model.parallel_tiling
|
||||||
seen["stack_tiling"] = vae.model.stack_tiling
|
|
||||||
# the guard never inspects the decoded shape, so keep it small
|
# the guard never inspects the decoded shape, so keep it small
|
||||||
return torch.zeros(1, 3, 2, 4, 4)
|
return torch.zeros(1, 3, 2, 4, 4)
|
||||||
|
|
||||||
@@ -242,15 +212,3 @@ def test_decode_latent_leaves_the_group_state_alone_when_the_grid_is_large_enoug
|
|||||||
assert seen["sp_enabled"] is True
|
assert seen["sp_enabled"] is True
|
||||||
assert seen["sp_process_group"] is state["sp_process_group"]
|
assert seen["sp_process_group"] is state["sp_process_group"]
|
||||||
assert seen["parallel_tiling"] is True
|
assert seen["parallel_tiling"] is True
|
||||||
|
|
||||||
|
|
||||||
def test_decode_latent_restores_native_stack_tiling_state():
|
|
||||||
vae, _, seen = _dispatch_vae(4)
|
|
||||||
vae.set_stack_tiling("auto")
|
|
||||||
# The adapter must not leak its per-request decision into the remote VAE.
|
|
||||||
vae.model.stack_tiling = True
|
|
||||||
|
|
||||||
MiniMaxH3VideoVAE.decode_latent(vae, _latent(16, 16))
|
|
||||||
|
|
||||||
assert seen["stack_tiling"] is False
|
|
||||||
assert vae.model.stack_tiling is True
|
|
||||||
@@ -395,28 +395,6 @@ def test_serve_cli_accepts_diffusion_pipeline_profiler_flag():
|
|||||||
assert stage_cfg["engine_args"]["enable_diffusion_pipeline_profiler"] is True
|
assert stage_cfg["engine_args"]["enable_diffusion_pipeline_profiler"] is True
|
||||||
|
|
||||||
|
|
||||||
def test_serve_cli_forwards_vae_stack_tiling():
|
|
||||||
parser = TrackingArgumentParser()
|
|
||||||
subparsers = parser.add_subparsers(dest="command")
|
|
||||||
OmniServeCommand().subparser_init(subparsers)
|
|
||||||
|
|
||||||
args = parser.parse_args(
|
|
||||||
[
|
|
||||||
"serve",
|
|
||||||
"MiniMaxAI/MiniMax-H3",
|
|
||||||
"--omni",
|
|
||||||
"--vae-stack-tiling",
|
|
||||||
"auto",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
explicit_kwargs = args.get_explicit_kwargs_dict()
|
|
||||||
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(explicit_kwargs)[0]
|
|
||||||
|
|
||||||
assert args.vae_stack_tiling == "auto"
|
|
||||||
assert stage_cfg["engine_args"]["vae_stack_tiling"] == "auto"
|
|
||||||
|
|
||||||
|
|
||||||
def test_serve_cli_forwards_distilled_lora_to_diffusion_stage():
|
def test_serve_cli_forwards_distilled_lora_to_diffusion_stage():
|
||||||
"""Ensure startup distilled LoRA options reach the online diffusion stage."""
|
"""Ensure startup distilled LoRA options reach the online diffusion stage."""
|
||||||
parser = TrackingArgumentParser()
|
parser = TrackingArgumentParser()
|
||||||
|
|||||||
@@ -711,7 +711,6 @@ class _DiffusionConfigProjection:
|
|||||||
fa_deterministic: bool = False
|
fa_deterministic: bool = False
|
||||||
vae_use_slicing: bool = False
|
vae_use_slicing: bool = False
|
||||||
vae_use_tiling: bool = False
|
vae_use_tiling: bool = False
|
||||||
vae_stack_tiling: Literal["auto", "true", "false"] = "false"
|
|
||||||
mask_strategy_file_path: str | None = None
|
mask_strategy_file_path: str | None = None
|
||||||
skip_time_steps: int = 15
|
skip_time_steps: int = 15
|
||||||
VSA_sparsity: float = 0.0
|
VSA_sparsity: float = 0.0
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from collections.abc import Callable, Mapping
|
|||||||
from dataclasses import asdict, dataclass, field, fields
|
from dataclasses import asdict, dataclass, field, fields
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Literal, NamedTuple
|
from typing import Any, NamedTuple
|
||||||
|
|
||||||
from transformers import PretrainedConfig
|
from transformers import PretrainedConfig
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
@@ -462,7 +462,6 @@ class StageDeployConfig:
|
|||||||
step_execution: bool | None = None
|
step_execution: bool | None = None
|
||||||
vae_use_slicing: bool | None = None
|
vae_use_slicing: bool | None = None
|
||||||
vae_use_tiling: bool | None = None
|
vae_use_tiling: bool | None = None
|
||||||
vae_stack_tiling: Literal["auto", "true", "false"] | None = None
|
|
||||||
boundary_ratio: float | None = None
|
boundary_ratio: float | None = None
|
||||||
flow_shift: float | None = None
|
flow_shift: float | None = None
|
||||||
diffusion_kv_cache_dtype: str | None = None
|
diffusion_kv_cache_dtype: str | None = None
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import random
|
|||||||
from collections.abc import Callable, Mapping
|
from collections.abc import Callable, Mapping
|
||||||
from dataclasses import dataclass, field, fields
|
from dataclasses import dataclass, field, fields
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import TYPE_CHECKING, Any, Literal
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import diffusers
|
import diffusers
|
||||||
import huggingface_hub
|
import huggingface_hub
|
||||||
@@ -831,7 +831,6 @@ class OmniDiffusionConfig:
|
|||||||
# VAE memory optimization parameters
|
# VAE memory optimization parameters
|
||||||
vae_use_slicing: bool = False
|
vae_use_slicing: bool = False
|
||||||
vae_use_tiling: bool = False
|
vae_use_tiling: bool = False
|
||||||
vae_stack_tiling: Literal["auto", "true", "false"] = "false"
|
|
||||||
|
|
||||||
# STA (Sliding Tile Attention) parameters
|
# STA (Sliding Tile Attention) parameters
|
||||||
mask_strategy_file_path: str | None = None
|
mask_strategy_file_path: str | None = None
|
||||||
|
|||||||
@@ -298,7 +298,7 @@ def _create_ltx25_natten_processor() -> LTX2VideoVaeNeighborhoodNattenProcessor:
|
|||||||
except (ImportError, OSError, RuntimeError, ValueError) as exc:
|
except (ImportError, OSError, RuntimeError, ValueError) as exc:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"LTX-2.5 DiffVAE requires the shi-labs/natten Hub kernel. "
|
"LTX-2.5 DiffVAE requires the shi-labs/natten Hub kernel. "
|
||||||
"Install kernels==0.14.1, use a supported GPU, leave "
|
"Install kernels==0.15.2, use a supported GPU, leave "
|
||||||
"DIFFUSERS_DISABLE_REMOTE_CODE unset, and allow Hub access during kernel initialization."
|
"DIFFUSERS_DISABLE_REMOTE_CODE unset, and allow Hub access during kernel initialization."
|
||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import json
|
|||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, ClassVar
|
from typing import Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.distributed as dist
|
import torch.distributed as dist
|
||||||
@@ -115,8 +115,6 @@ class _AudioVAEDeterminismContext(AbstractContextManager):
|
|||||||
class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||||
"""Adapter around the checkpoint's native parallel-tiled video VAE."""
|
"""Adapter around the checkpoint's native parallel-tiled video VAE."""
|
||||||
|
|
||||||
supports_stack_tiling: ClassVar[bool] = True
|
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
component_path: str,
|
component_path: str,
|
||||||
@@ -155,30 +153,6 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
|||||||
self.use_slicing = False
|
self.use_slicing = False
|
||||||
self.parallel_size = 1
|
self.parallel_size = 1
|
||||||
self.device_module = torch.get_device_module()
|
self.device_module = torch.get_device_module()
|
||||||
self.stack_tiling_mode = "false"
|
|
||||||
|
|
||||||
def set_stack_tiling(self, mode: str | bool) -> None:
|
|
||||||
"""Configure native decoder tile batching.
|
|
||||||
|
|
||||||
``auto`` is resolved per decode because the useful batch size depends
|
|
||||||
on both the request's tile grid and the VAE parallel group size.
|
|
||||||
"""
|
|
||||||
if not hasattr(self.model, "stack_tiling"):
|
|
||||||
raise RuntimeError("MiniMax H3 video VAE remote model does not expose stack_tiling")
|
|
||||||
if isinstance(mode, bool):
|
|
||||||
mode = "true" if mode else "false"
|
|
||||||
if mode not in ("auto", "true", "false"):
|
|
||||||
raise ValueError(f"invalid VAE stacked-tiling mode: {mode!r}")
|
|
||||||
self.stack_tiling_mode = mode
|
|
||||||
self.model.stack_tiling = mode == "true"
|
|
||||||
|
|
||||||
def _should_stack_decode_tiles(self, latent: torch.Tensor) -> bool:
|
|
||||||
if self.stack_tiling_mode != "auto":
|
|
||||||
return self.stack_tiling_mode == "true"
|
|
||||||
# Stacking is worthwhile when every rank has more than one tile in
|
|
||||||
# the common round-robin distribution. The tile grid is global and
|
|
||||||
# therefore identical on all ranks.
|
|
||||||
return self._decoder_tile_count(latent) >= 2 * self.parallel_size
|
|
||||||
|
|
||||||
def load_to_device(self) -> None:
|
def load_to_device(self) -> None:
|
||||||
if self._stager is not None:
|
if self._stager is not None:
|
||||||
@@ -414,18 +388,7 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
|||||||
tiling_context = nullcontext()
|
tiling_context = nullcontext()
|
||||||
|
|
||||||
with tiling_context:
|
with tiling_context:
|
||||||
stack_tiling_mode = getattr(self, "stack_tiling_mode", "false")
|
decoded = self.model.decode_base(latent * std + mean)
|
||||||
if stack_tiling_mode == "false" and not hasattr(self.model, "stack_tiling"):
|
|
||||||
# Keep the default path compatible with older H3 snapshots
|
|
||||||
# that predate the native stacked-tiling attribute.
|
|
||||||
decoded = self.model.decode_base(latent * std + mean)
|
|
||||||
else:
|
|
||||||
previous_stack_tiling = self.model.stack_tiling
|
|
||||||
self.model.stack_tiling = self._should_stack_decode_tiles(latent)
|
|
||||||
try:
|
|
||||||
decoded = self.model.decode_base(latent * std + mean)
|
|
||||||
finally:
|
|
||||||
self.model.stack_tiling = previous_stack_tiling
|
|
||||||
frames = self.model.processor.revert_tensor(decoded)
|
frames = self.model.processor.revert_tensor(decoded)
|
||||||
if frames.ndim == 4:
|
if frames.ndim == 4:
|
||||||
frames = frames.unsqueeze(0).transpose(1, 2)
|
frames = frames.unsqueeze(0).transpose(1, 2)
|
||||||
|
|||||||
@@ -387,38 +387,6 @@ def _prepare_diffusion_quant_config(
|
|||||||
configure_quant_config(quant_config, model_class)
|
configure_quant_config(quant_config, model_class)
|
||||||
|
|
||||||
|
|
||||||
def _configure_vae_stack_tiling(model: nn.Module, od_config: OmniDiffusionConfig) -> None:
|
|
||||||
"""Configure optional native batching of VAE decoder tiles."""
|
|
||||||
mode = getattr(od_config, "vae_stack_tiling", "false")
|
|
||||||
if isinstance(mode, bool):
|
|
||||||
mode = "true" if mode else "false"
|
|
||||||
if mode not in ("auto", "true", "false"):
|
|
||||||
raise ValueError(f"invalid vae_stack_tiling mode: {mode!r}")
|
|
||||||
if mode == "false":
|
|
||||||
return
|
|
||||||
|
|
||||||
vae = getattr(model, "vae", None)
|
|
||||||
if vae is None or not getattr(vae, "supports_stack_tiling", False):
|
|
||||||
if mode == "auto":
|
|
||||||
logger.info(
|
|
||||||
"VAE stacked tiling auto mode is unavailable for %s; using sequential tiles.",
|
|
||||||
od_config.model_class_name,
|
|
||||||
)
|
|
||||||
return
|
|
||||||
raise ValueError(
|
|
||||||
f"vae_stack_tiling was requested, but {od_config.model_class_name} does not declare stacked-tiling support"
|
|
||||||
)
|
|
||||||
setter = getattr(vae, "set_stack_tiling", None)
|
|
||||||
if not callable(setter):
|
|
||||||
raise ValueError("vae_stack_tiling was requested, but the VAE does not provide set_stack_tiling()")
|
|
||||||
|
|
||||||
if not od_config.vae_use_tiling:
|
|
||||||
logger.info("vae_stack_tiling requires vae_use_tiling; automatically enabling it.")
|
|
||||||
od_config.vae_use_tiling = True
|
|
||||||
setter(mode)
|
|
||||||
logger.info("VAE stacked tiling mode set to %s for %s.", mode, od_config.model_class_name)
|
|
||||||
|
|
||||||
|
|
||||||
def initialize_model(
|
def initialize_model(
|
||||||
od_config: OmniDiffusionConfig,
|
od_config: OmniDiffusionConfig,
|
||||||
) -> nn.Module:
|
) -> nn.Module:
|
||||||
@@ -460,8 +428,6 @@ def initialize_model(
|
|||||||
)
|
)
|
||||||
od_config.vae_use_tiling = True
|
od_config.vae_use_tiling = True
|
||||||
|
|
||||||
_configure_vae_stack_tiling(model, od_config)
|
|
||||||
|
|
||||||
# Configure VAE memory optimization settings from config
|
# Configure VAE memory optimization settings from config
|
||||||
if hasattr(model, "vae") and hasattr(model.vae, "use_slicing"):
|
if hasattr(model, "vae") and hasattr(model.vae, "use_slicing"):
|
||||||
model.vae.use_slicing = od_config.vae_use_slicing
|
model.vae.use_slicing = od_config.vae_use_slicing
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import os
|
|||||||
import tempfile
|
import tempfile
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass, field, fields
|
from dataclasses import dataclass, field, fields
|
||||||
from typing import Any, Literal, cast
|
from typing import Any, cast
|
||||||
|
|
||||||
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
@@ -566,7 +566,6 @@ class OrchestratorArgs:
|
|||||||
step_execution: bool = False
|
step_execution: bool = False
|
||||||
vae_use_slicing: bool = False
|
vae_use_slicing: bool = False
|
||||||
vae_use_tiling: bool = False
|
vae_use_tiling: bool = False
|
||||||
vae_stack_tiling: Literal["auto", "true", "false"] = "false"
|
|
||||||
enable_multithread_weight_load: bool = True
|
enable_multithread_weight_load: bool = True
|
||||||
num_weight_load_threads: int = 4
|
num_weight_load_threads: int = 4
|
||||||
enable_cpu_offload: bool = False
|
enable_cpu_offload: bool = False
|
||||||
|
|||||||
@@ -1132,7 +1132,6 @@ class AsyncOmniEngine:
|
|||||||
"request_batch_max_wait_ms": kwargs.get("request_batch_max_wait_ms", 0.0),
|
"request_batch_max_wait_ms": kwargs.get("request_batch_max_wait_ms", 0.0),
|
||||||
"vae_use_slicing": kwargs.get("vae_use_slicing", False),
|
"vae_use_slicing": kwargs.get("vae_use_slicing", False),
|
||||||
"vae_use_tiling": kwargs.get("vae_use_tiling", False),
|
"vae_use_tiling": kwargs.get("vae_use_tiling", False),
|
||||||
"vae_stack_tiling": kwargs.get("vae_stack_tiling", "false"),
|
|
||||||
"cache_backend": cache_backend,
|
"cache_backend": cache_backend,
|
||||||
"cache_config": cache_config,
|
"cache_config": cache_config,
|
||||||
"enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False),
|
"enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False),
|
||||||
|
|||||||
@@ -686,15 +686,6 @@ class OmniServeCommand(CLISubcommand):
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="Enable VAE tiling for memory optimization (useful for mitigating OOM issues).",
|
help="Enable VAE tiling for memory optimization (useful for mitigating OOM issues).",
|
||||||
)
|
)
|
||||||
omni_config_group.add_argument(
|
|
||||||
"--vae-stack-tiling",
|
|
||||||
choices=("auto", "true", "false"),
|
|
||||||
default="false",
|
|
||||||
help=(
|
|
||||||
"Batch native VAE tiles into fewer model calls. 'auto' enables it "
|
|
||||||
"when each VAE-parallel rank has multiple tiles (default: false)."
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Parallel weight loading (faster diffusion startup)
|
# Parallel weight loading (faster diffusion startup)
|
||||||
omni_config_group.add_argument(
|
omni_config_group.add_argument(
|
||||||
|
|||||||
Reference in new issue
Block a user