[New model] Qwen-Image-2.1 support (rebase + review follow-ups) (#8099)

Signed-off-by: NumberWan <wantszkin2003@gmail.com>
Signed-off-by: SamitHuang <285365963@qq.com>
Signed-off-by: Samit <285365963@qq.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: SamitHuang <285365963@qq.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
authored and GitHub committed 2026-10-07 21:23:59 +08:00
1 parent 8ef77be66e
commit 3dc35694b3
52 files changed
+8058 -136

No files matched your search

@@ -98,6 +98,12 @@ class PinnedModuleStager:
A module iterable is treated as one staging group. It uses one copy stream
and one reusable completion event. Tensors sharing storage keep their
shapes, strides, offsets, dtypes, and aliases across every transition.
With ``retain_device_storage`` the device storages survive ``offload``:
``load`` then only rebinds to the same fixed storages (weights are
immutable, so no H2D copy repeats). The stable storage addresses keep
CUDA-graph-captured weight pointers valid across offload/load swaps, at
the cost of keeping the device storage resident while offloaded.
"""
def __init__(
@@ -108,6 +114,7 @@ class PinnedModuleStager:
pin_memory: bool = True,
copy_stream: Any | None = None,
cache_retention: BoundedAllocatorCache | None = None,
retain_device_storage: bool = False,
) -> None:
modules = (module,) if isinstance(module, nn.Module) else tuple(module)
if not modules or not all(isinstance(item, nn.Module) for item in modules):
@@ -117,6 +124,7 @@ class PinnedModuleStager:
self.copy_stream = copy_stream if copy_stream is not None else current_omni_platform.Stream()
self._ready_event = current_omni_platform.Event()
self.cache_retention = cache_retention
self.retain_device_storage = retain_device_storage
self.loaded = False
self._groups = self._snapshot_groups(modules, pin_memory=pin_memory)
self._device_storages: list[torch.Tensor] = []
@@ -234,6 +242,12 @@ class PinnedModuleStager:
def load(self) -> None:
if self.loaded:
return
if self._device_storages:
# Storage retained from a previous load still holds the immutable
# weights; rebinding suffices, no H2D copy repeats.
self._bind(self._device_storages)
self.loaded = True
return
try:
self._load_once()
except torch.OutOfMemoryError:
@@ -263,7 +277,8 @@ class PinnedModuleStager:
self.loaded = False
self._release_cache(force=True)
raise
self._device_storages.clear()
if not self.retain_device_storage:
self._device_storages.clear()
self.loaded = False
self._release_cache()
@@ -13,6 +13,7 @@ from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
from vllm_omni.platforms import current_omni_platform
from .base import OffloadBackend, OffloadConfig, SupportsModelCpuOffload
from .module_residency import PinnedModuleStager
from .plan_resolver import resolve_offload_plan
logger = init_logger(__name__)
@@ -60,6 +61,7 @@ class SequentialOffloadHook(ModelHook):
pin_memory: bool = True,
use_hsdp: bool = False,
offload_after_context: bool = True,
persistent_staging: bool = False,
):
# Modules to offload to CPU before this module runs
self.offload_targets = offload_targets
@@ -67,6 +69,21 @@ class SequentialOffloadHook(ModelHook):
self.pin_memory = pin_memory
self.use_hsdp = use_hsdp
self.offload_after_context = offload_after_context
self.persistent_staging = persistent_staging
self._stager: PinnedModuleStager | None = None
def initialize_hook(self, module: nn.Module) -> nn.Module:
if self.persistent_staging and not self.use_hsdp:
# Fixed staging storage keeps CUDA-graph-captured weight pointers
# valid across offload/load swaps. HSDP keeps the plain move
# semantics (decode graphs refuse HSDP anyway).
self._stager = PinnedModuleStager(
module,
self.device,
pin_memory=self.pin_memory,
retain_device_storage=True,
)
return module
@staticmethod
def _move_params(
@@ -101,7 +118,23 @@ class SequentialOffloadHook(ModelHook):
moved = True
return moved
@staticmethod
def _module_stager(module: nn.Module) -> PinnedModuleStager | None:
"""The persistent stager owned by ``module``'s own sequential hook, if any.
Cross-module swaps call ``_to_cpu(target)`` through the *running*
module's hook, so staging state must be resolved on the target's hook
to keep its ``loaded`` bookkeeping consistent.
"""
registry: HookRegistry | None = getattr(module, "_hook_registry", None)
hook = registry.get_hook(SequentialOffloadHook._HOOK_NAME) if registry is not None else None
return hook._stager if isinstance(hook, SequentialOffloadHook) else None
def _to_cpu(self, module: nn.Module) -> None:
stager = self._module_stager(module)
if stager is not None:
stager.offload()
return
# XPU's allocator doesn't respect stream dependencies in empty_cache,
# so non-blocking copies can race with cache eviction. Use blocking
# copies on XPU to avoid NULL pointer errors during DMA.
@@ -116,6 +149,10 @@ class SequentialOffloadHook(ModelHook):
current_omni_platform.empty_cache()
def _to_gpu(self, module: nn.Module) -> None:
stager = self._module_stager(module)
if stager is not None:
stager.load()
return
self._move_params(module, self.device, non_blocking=False)
def pre_forward(self, module: nn.Module, *args, **kwargs) -> tuple[tuple, dict]:
@@ -147,6 +184,7 @@ def apply_sequential_offload(
offload_initial_dits: bool = False,
offload_dit_modules: Collection[nn.Module] | None = None,
offload_encoder_modules: Collection[nn.Module] | None = None,
persistent_dit_staging: bool = False,
) -> None:
"""Apply sequential offloading hooks to DiT and encoder modules.
@@ -165,6 +203,10 @@ def apply_sequential_offload(
every DiT for backward compatibility.
offload_encoder_modules: Encoder/stage modules allowed to move to CPU.
None selects every supplied module for backward compatibility.
persistent_dit_staging: Keep DiT weights on fixed device staging
storage across swaps, so CUDA-graph-captured weight pointers stay
valid. Encoders keep plain move semantics so their memory is
actually freed while the DiT runs.
Example:
>>> apply_sequential_offload(
@@ -193,6 +235,7 @@ def apply_sequential_offload(
pin_memory=pin_memory,
use_hsdp=use_hsdp,
offload_after_context=id(dit_mod) in selected_dit_ids,
persistent_staging=persistent_dit_staging,
)
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
logger.debug("Registered offload hook for %s", dit_mod.__class__.__name__)
@@ -261,6 +304,15 @@ def _get_sequential_offload_hook(module: nn.Module) -> SequentialOffloadHook:
return hook
def sequential_offload_staging_active(module: nn.Module) -> bool:
"""True when the module's sequential offload hook serves fixed device storage.
Fixed staging storage keeps CUDA-graph-captured weight pointers valid
across offload/load swaps.
"""
return SequentialOffloadHook._module_stager(module) is not None
@contextmanager
def sequential_offload_component(module: nn.Module) -> Iterator[None]:
"""Activate and release a hooked component called outside ``forward``."""
@@ -362,6 +414,14 @@ class ModelLevelOffloadBackend(OffloadBackend):
use_hsdp=self.config.use_hsdp,
offload_dit_modules=selected_dits,
offload_encoder_modules=selected_encoders,
# Fixed DiT staging keeps decode-graph weight pointers valid;
# only models that capture weight pointers (Qwen-Image-2.1's
# CUDA-graph decode) pay for it — every other model keeps
# plain move semantics so offloading actually frees DiT VRAM
# while the encoders run.
persistent_dit_staging=(
self.device.type == "cuda" and any(getattr(dit, "enable_cuda_graph_decode", False) for dit in dits)
),
)
except BaseException:
try: