[New model] Qwen-Image-2.1 support (rebase + review follow-ups) (#8099)

Signed-off-by: NumberWan <wantszkin2003@gmail.com>
Signed-off-by: SamitHuang <285365963@qq.com>
Signed-off-by: Samit <285365963@qq.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: SamitHuang <285365963@qq.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
authored and GitHub committed 2026-10-07 21:23:59 +08:00
1 parent 8ef77be66e
commit 3dc35694b3
52 files changed
+8058 -136

No files matched your search

@@ -10,11 +10,13 @@ from vllm_omni.diffusion.attention.backends.abstract import (
AttentionMetadata,
)
from vllm_omni.diffusion.attention.backends.sdpa import _maybe_reshape_attn_mask
from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import piecewise_attn
class CuDNNAttentionBackend(AttentionBackend):
accept_output_buffer: bool = True
supports_prefix_kv_slicing: bool = True
supports_piecewise_spans: bool = True
# cuDNN 9.5+ FMHA on Blackwell: head_dim divisible by 8 and at most 256
# for BF16/FP16. Used by automatic platform selection; explicit CUDNN_ATTN
@@ -74,6 +76,23 @@ class CuDNNAttentionImpl(AttentionImpl):
value: torch.Tensor,
attn_metadata: AttentionMetadata | None = None,
) -> torch.Tensor:
if (
attn_metadata is not None
and attn_metadata.full_attn_spans is not None
and (attn_metadata.attn_mask is None or attn_metadata.attn_mask.ndim != 4)
):
if attn_metadata.attn_mask is not None or "valid_kv_length" in attn_metadata.extra:
raise ValueError("Piecewise cuDNN attention cannot combine spans with a padding mask.")
return piecewise_attn(
query,
key,
value,
attn_metadata.full_attn_spans,
self.softmax_scale,
self._piecewise_attention,
query_ranges=attn_metadata.query_ranges,
)
attention_mask = None
if attn_metadata:
valid_kv_length = attn_metadata.extra.get("valid_kv_length")
@@ -97,6 +116,19 @@ class CuDNNAttentionImpl(AttentionImpl):
mask_mode="broadcast_k",
)
return self._attention(query, key, value, attention_mask, self.causal, self.softmax_scale)
def _piecewise_attention(self, query, key, value, *, causal, softmax_scale):
mask = None
if causal:
# SDPA's causal flag aligns at the top left. Text segments need
# bottom-right alignment so every query can also see its prefix.
q_len, kv_len = query.shape[1], key.shape[1]
mask = torch.ones(q_len, kv_len, dtype=torch.bool, device=query.device).tril(kv_len - q_len)
mask = mask[None, None]
return self._attention(query, key, value, mask, False, softmax_scale)
def _attention(self, query, key, value, attention_mask, causal, softmax_scale):
enable_gqa = query.shape[2] != key.shape[2]
kv_seq_len = key.shape[1]
query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value))
@@ -130,8 +162,8 @@ class CuDNNAttentionImpl(AttentionImpl):
value,
attn_mask=attention_mask,
dropout_p=0.0,
is_causal=self.causal,
scale=self.softmax_scale,
is_causal=causal,
scale=softmax_scale,
enable_gqa=enable_gqa,
)
return output.permute(0, 2, 1, 3)
+15 -2
View File
@@ -16,7 +16,7 @@ import diffusers
import huggingface_hub
import torch
from PIL import Image
from pydantic import Field, model_validator
from pydantic import model_validator
from typing_extensions import Self
from vllm.config.utils import config
from vllm.logger import init_logger
@@ -981,6 +981,19 @@ class OmniDiffusionConfig:
# Compilation
enforce_eager: bool = False
# Capture fixed-shape KV-cache decode (denoising) steps into CUDA graphs.
# Currently only Qwen-Image-2.1's transformer implements this; other models
# ignore the flag. This is a per-model diffusion path, independent of the
# AR engine's ``compilation_config.cudagraph_mode`` (which does not apply
# to diffusion stages). It stacks with ``diffusion_compile_granularity``:
# the DiT blocks are still torch.compile'd and graph capture records the
# compiled (fused) kernels, while inductor's own cudagraphs stay off.
# ``enforce_eager=True`` forces eager decode and
# disables capture regardless of this flag; unsupported configurations
# (SP/TP, ring, HSDP, dynamic LoRA, padded masks, quantized prefix KV, or
# a second in-flight request aliasing the same graph key) log and fall
# back to eager decode.
enable_cuda_graph_decode: bool = True
# Controls the generic compilation path used when a pipeline does not
# provide its own setup_compile() implementation.
diffusion_compile_granularity: str = "regional"
@@ -1140,7 +1153,7 @@ class OmniDiffusionConfig:
request_batch_max_wait_ms: float = 0.0
# Supplementary model specific parameters
extras: dict[str, Any] = Field(default_factory=dict)
extras: dict[str, Any] = field(default_factory=dict)
@property
def is_moe(self) -> bool:
@@ -0,0 +1,149 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from typing import Any
import torch
from diffusers.models.autoencoders.vae import DecoderOutput
from vllm.logger import init_logger
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
DistributedOperator,
DistributedVaeMixin,
GridSpec,
TileTask,
)
from vllm_omni.diffusion.models.qwen_image_21.autoencoder_kl_qwenimage21 import (
AutoencoderKLQwenImage21,
_cudnn_deterministic,
_unpatchify,
)
logger = init_logger(__name__)
class DistributedAutoencoderKLQwenImage21(AutoencoderKLQwenImage21, DistributedVaeMixin):
@classmethod
def from_pretrained(cls, *args: Any, **kwargs: Any):
model = super().from_pretrained(*args, **kwargs)
model.init_distributed()
return model
def tile_split(self, z: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
# mostly copy from AutoencoderKLQwenImage21.tiled_decode
_, _, num_frames, height, width = z.shape
sample_height = height * self.spatial_compression_ratio
sample_width = width * self.spatial_compression_ratio
tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio
tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio
tile_latent_stride_height = self.tile_sample_stride_height // self.spatial_compression_ratio
tile_latent_stride_width = self.tile_sample_stride_width // self.spatial_compression_ratio
tile_sample_stride_height = self.tile_sample_stride_height
tile_sample_stride_width = self.tile_sample_stride_width
if self.config.patch_size is not None:
sample_height = sample_height // self.config.patch_size
sample_width = sample_width // self.config.patch_size
tile_sample_stride_height = tile_sample_stride_height // self.config.patch_size
tile_sample_stride_width = tile_sample_stride_width // self.config.patch_size
blend_height = self.tile_sample_min_height // self.config.patch_size - tile_sample_stride_height
blend_width = self.tile_sample_min_width // self.config.patch_size - tile_sample_stride_width
else:
blend_height = self.tile_sample_min_height - tile_sample_stride_height
blend_width = self.tile_sample_min_width - tile_sample_stride_width
# Split z into overlapping tiles and decode them separately.
# The tiles have an overlap to avoid seams between tiles.
tiletask_list: list[TileTask] = []
for i in range(0, height, tile_latent_stride_height):
for j in range(0, width, tile_latent_stride_width):
time_list = []
for k in range(num_frames):
tile = z[:, :, k : k + 1, i : i + tile_latent_min_height, j : j + tile_latent_min_width]
time_list.append(tile)
tiletask_list.append(
TileTask(
len(tiletask_list),
(i // tile_latent_stride_height, j // tile_latent_stride_width),
time_list,
workload=time_list[0].shape[3] * time_list[0].shape[4],
)
)
tile_spec = {
"sample_height": sample_height,
"sample_width": sample_width,
"blend_height": blend_height,
"blend_width": blend_width,
"tile_sample_stride_height": tile_sample_stride_height,
"tile_sample_stride_width": tile_sample_stride_width,
}
grid_spec = GridSpec(
split_dims=(3, 4),
grid_shape=(tiletask_list[-1].grid_coord[0] + 1, tiletask_list[-1].grid_coord[1] + 1),
tile_spec=tile_spec,
output_dtype=self.dtype,
)
return tiletask_list, grid_spec
def tile_exec(self, task: TileTask) -> torch.Tensor:
"""Decode a single latent tile into RGB space."""
self.clear_cache()
time = []
with _cudnn_deterministic():
for k in range(len(task.tensor)):
self._conv_idx = [0]
tile = self.post_quant_conv(task.tensor[k])
decoded = self.decoder(tile, feat_cache=self._feat_map, feat_idx=self._conv_idx, first_chunk=(k == 0))
time.append(decoded)
result = torch.cat(time, dim=2)
return result
def tile_merge(self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid_spec: GridSpec) -> torch.Tensor:
"""Merge decoded tiles into a full image."""
grid_h, grid_w = grid_spec.grid_shape
self.clear_cache()
result_rows = []
for i in range(grid_h):
result_row = []
for j in range(grid_w):
tile = coord_tensor_map[(i, j)]
if i > 0:
tile = self.blend_v(coord_tensor_map[(i - 1, j)], tile, grid_spec.tile_spec["blend_height"])
if j > 0:
tile = self.blend_h(coord_tensor_map[(i, j - 1)], tile, grid_spec.tile_spec["blend_width"])
result_row.append(
tile[
:,
:,
:,
: grid_spec.tile_spec["tile_sample_stride_height"],
: grid_spec.tile_spec["tile_sample_stride_width"],
]
)
result_rows.append(torch.cat(result_row, dim=-1))
dec = torch.cat(result_rows, dim=3)[
:, :, :, : grid_spec.tile_spec["sample_height"], : grid_spec.tile_spec["sample_width"]
]
if self.config.patch_size is not None:
dec = _unpatchify(dec, patch_size=self.config.patch_size)
dec = torch.clamp(dec, min=-1.0, max=1.0)
return dec
def tiled_decode(self, z: torch.Tensor, return_dict: bool = True):
if not self.is_distributed_enabled():
return super().tiled_decode(z, return_dict=return_dict)
logger.debug("Decode running with distributed executor")
result = self.distributed_executor.execute(
z,
DistributedOperator(split=self.tile_split, exec=self.tile_exec, merge=self.tile_merge),
broadcast_result=True,
)
if not return_dict:
return (result,)
return DecoderOutput(sample=result)
@@ -3,6 +3,8 @@
from __future__ import annotations
from collections.abc import Callable
from contextlib import contextmanager
from dataclasses import dataclass, field
import torch
@@ -34,9 +36,9 @@ class TileTask:
@dataclass
class DistributedOperator:
split: callable
exec: callable
merge: callable
split: Callable[[torch.Tensor], tuple[list[TileTask], GridSpec]]
exec: Callable[[TileTask], torch.Tensor]
merge: Callable[[dict[tuple[int, ...], torch.Tensor], GridSpec], torch.Tensor]
class DistributedVaeExecutor:
@@ -46,7 +48,11 @@ class DistributedVaeExecutor:
def __init__(self):
# Use a dedicated process group spanning the complete worker WORLD.
self.group = get_world_group().device_group
world_group = get_world_group()
self.group = world_group.device_group
self._error_group = world_group.cpu_group
# Reporting an OOM must not require another device allocation.
self._error_status = torch.zeros((), dtype=torch.int32, device="cpu")
self.world_size = dist.get_world_size(self.group)
self.rank = dist.get_rank(self.group)
self.parallel_size = 1
@@ -56,8 +62,34 @@ class DistributedVaeExecutor:
self.parallel_size = parallel_size
self.parallel_mode = mode
@contextmanager
def _sync_errors(self, stage: str):
"""Agree on local-stage failures before entering the next collective."""
status = 0
message = ""
try:
yield
except torch.OutOfMemoryError as error:
status = 1
message = str(error)
except Exception as error:
# A non-retryable error takes precedence over OOM on another rank.
status = 2
message = f"{type(error).__name__}: {error}"
self._error_status.fill_(status)
dist.all_reduce(self._error_status, op=dist.ReduceOp.MAX, group=self._error_group)
global_status = self._error_status.item()
if global_status == 2:
detail = message if status == 2 else "another rank failed"
raise RuntimeError(f"Distributed VAE {stage} failed: {detail}")
if global_status == 1:
detail = message if status == 1 else "another rank ran out of memory"
raise torch.OutOfMemoryError(f"Distributed VAE {stage} failed: {detail}")
def gather_tensors(self, tensor: torch.Tensor):
gather_list = [torch.empty_like(tensor) for _ in range(self.world_size)]
with self._sync_errors("gather allocation"):
gather_list = [torch.empty_like(tensor) for _ in range(self.world_size)]
dist.all_gather(gather_list, tensor, group=self.group)
return gather_list if self.rank == 0 else None
@@ -66,11 +98,12 @@ class DistributedVaeExecutor:
return tensor
def _compute_global_padding_shape(self, local_results, output_ndim: int, device):
local_tile_max_dims = [0] * output_ndim
for _, tile_tensor in local_results:
for dim_idx, dim_size in enumerate(tile_tensor.shape):
local_tile_max_dims[dim_idx] = max(local_tile_max_dims[dim_idx], dim_size)
local_shape_stat = torch.tensor([len(local_results), *local_tile_max_dims], device=device)
with self._sync_errors("shape preparation"):
local_tile_max_dims = [0] * output_ndim
for _, tile_tensor in local_results:
for dim_idx, dim_size in enumerate(tile_tensor.shape):
local_tile_max_dims[dim_idx] = max(local_tile_max_dims[dim_idx], dim_size)
local_shape_stat = torch.tensor([len(local_results), *local_tile_max_dims], device=device)
dist.all_reduce(
local_shape_stat,
op=dist.ReduceOp.MAX,
@@ -100,17 +133,16 @@ class DistributedVaeExecutor:
tid = int(meta_src[idx, 0])
if tid < 0:
continue
slices = [slice(None)] * tiles_src[idx].ndim
slices_list = [slice(None)] * tiles_src[idx].ndim
for i, dim in enumerate(grid_spec.split_dims):
slices[dim] = slice(0, int(meta_src[idx, i + 1]))
slices = tuple(slices)
coord_tensor_map[tid_coord_map[tid]] = tiles_src[idx][slices]
slices_list[dim] = slice(0, int(meta_src[idx, i + 1]))
coord_tensor_map[tid_coord_map[tid]] = tiles_src[idx][tuple(slices_list)]
return coord_tensor_map
def _balance_tasks(self, task_list, num_rank):
workloads = [0] * num_rank
assigned = [[] for _ in range(num_rank)]
assigned: list[list[TileTask]] = [[] for _ in range(num_rank)]
for task in sorted(task_list, key=lambda t: t.workload, reverse=True):
r = workloads.index(min(workloads))
@@ -122,49 +154,51 @@ class DistributedVaeExecutor:
def execute(self, z: torch.Tensor, operator: DistributedOperator, broadcast_result: bool = True):
pp_size = min(self.parallel_size, self.world_size)
# 1. Split into tiles
tiletask_list, grid_spec = operator.split(z)
tid_coord_map = {task.tile_id: task.grid_coord for task in tiletask_list}
# 2. local decode
assigned = self._balance_tasks(tiletask_list, pp_size)
local_tasks = assigned[self.rank] if self.rank < pp_size else []
local_results = [(t.tile_id, operator.exec(t)) for t in local_tasks]
with self._sync_errors("tile decode"):
tiletask_list, grid_spec = operator.split(z)
tid_coord_map = {task.tile_id: task.grid_coord for task in tiletask_list}
assigned = self._balance_tasks(tiletask_list, pp_size)
local_tasks = assigned[self.rank] if self.rank < pp_size else []
local_results = [(t.tile_id, operator.exec(t)) for t in local_tasks]
# 3. compute shape per rank
global_padding_shape = self._compute_global_padding_shape(local_results, z.ndim, z.device)
# 5. prepare tile tensors
output_dtype = grid_spec.output_dtype if grid_spec.output_dtype is not None else z.dtype
local_tile_tensor, local_meta_tensor = self._pack_local_tiles(
local_results, global_padding_shape, grid_spec, z.device, output_dtype
)
with self._sync_errors("tile packing"):
local_tile_tensor, local_meta_tensor = self._pack_local_tiles(
local_results, global_padding_shape, grid_spec, z.device, output_dtype
)
# 6. gather tiles & meta
meta_gather = self.gather_tensors(local_meta_tensor)
tile_gather = self.gather_tensors(local_tile_tensor)
if self.rank != 0:
result = torch.empty(0, device=z.device) # Dummy return for non-zero ranks
else:
# 7. reconstruct full tensor (rank 0)
coord_tensor_map = self._unpack_tiles(meta_gather, tile_gather, grid_spec, tid_coord_map)
result = operator.merge(coord_tensor_map, grid_spec)
with self._sync_errors("tile merge"):
if self.rank != 0:
result = torch.empty(0, device=z.device) # Dummy return for non-zero ranks
else:
# 7. reconstruct full tensor (rank 0)
coord_tensor_map = self._unpack_tiles(meta_gather, tile_gather, grid_spec, tid_coord_map)
result = operator.merge(coord_tensor_map, grid_spec)
if broadcast_result:
result = self._sync_final_result(result, z.ndim, z.device, output_dtype)
return result
def _sync_final_result(self, rank0_result, output_ndim, output_device, output_dtype):
shape_tensor = torch.empty((output_ndim,), device=output_device, dtype=torch.int64)
if self.rank == 0:
shape_tensor.copy_(torch.tensor(tuple(rank0_result.shape), device=output_device, dtype=torch.int64))
with self._sync_errors("output shape preparation"):
shape_tensor = torch.empty((output_ndim,), device=output_device, dtype=torch.int64)
if self.rank == 0:
shape_tensor.copy_(torch.tensor(tuple(rank0_result.shape), device=output_device, dtype=torch.int64))
dist.broadcast(shape_tensor, src=0, group=self.group)
if self.rank != 0:
sync_result = torch.empty(tuple(shape_tensor.tolist()), device=output_device, dtype=output_dtype)
else:
sync_result = rank0_result
with self._sync_errors("output allocation"):
if self.rank != 0:
sync_result = torch.empty(tuple(shape_tensor.tolist()), device=output_device, dtype=output_dtype)
else:
sync_result = rank0_result
dist.broadcast(sync_result, src=0, group=self.group)
return sync_result
+5
View File
@@ -23,6 +23,11 @@ lora_convert_mapping: dict[str, Callable] = {
"QwenImagePipeline": _convert_non_diffusers_qwen_lora_to_diffusers,
"QwenImageEditPipeline": _convert_non_diffusers_qwen_lora_to_diffusers,
"QwenImageEditPlusPipeline": _convert_non_diffusers_qwen_lora_to_diffusers,
# 2.1's single-stream blocks keep the same leaf names the converter's
# protected n-grams cover (`attn.to_q/to_k/to_v`, `img_mlp`, ...), so the
# 2.0 converter applies. Non-diffusers LoRAs that target the fused `to_qkv`
# projection are not supported (the converter would split it into `to.qkv`).
"QwenImage21Pipeline": _convert_non_diffusers_qwen_lora_to_diffusers,
"Wan22Pipeline": _convert_non_diffusers_wan_lora_to_diffusers,
"Wan22I2VPipeline": _convert_non_diffusers_wan_lora_to_diffusers,
}
+8
View File
@@ -26,6 +26,8 @@ class DiffusionModelMetadata:
# FLUX.2 Klein supports up to four reference images.
FLUX2_KLEIN_MAX_INPUT_IMAGES = 4
QWEN_IMAGE_EDIT_PLUS_MAX_INPUT_IMAGES = 4
# Qwen-Image 2.1 image-conditioned generation caps condition images at 4.
QWEN_IMAGE_21_MAX_INPUT_IMAGES = 4
# Upstream HunyuanImage-3.0 "Multi-Image Fusion" caps reference images at 3.
HUNYUAN_IMAGE3_MAX_INPUT_IMAGES = 3
# Boogu-Image editing (TI2I) supports a single reference image for now.
@@ -42,6 +44,12 @@ _DIFFUSION_MODEL_METADATA: dict[str, DiffusionModelMetadata] = {
supports_multimodal_inputs=True,
max_multimodal_image_inputs=QWEN_IMAGE_EDIT_PLUS_MAX_INPUT_IMAGES,
),
# Qwen-Image 2.1 handles both text-to-image (no image) and
# image-conditioned requests through the same pipeline class.
"QwenImage21Pipeline": DiffusionModelMetadata(
supports_multimodal_inputs=True,
max_multimodal_image_inputs=QWEN_IMAGE_21_MAX_INPUT_IMAGES,
),
"HunyuanImage3Pipeline": DiffusionModelMetadata(
supports_multimodal_inputs=True,
max_multimodal_image_inputs=HUNYUAN_IMAGE3_MAX_INPUT_IMAGES,
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Qwen Image 2.1 diffusion model components.
Lazy by necessity: the pipeline imports the distributed VAE
(`vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_qwenimage21`),
which in turn imports the plain VAE module from this package. Eager imports
here would make that a circular import whenever the distributed module is
imported first.
"""
from __future__ import annotations
from typing import Any
__all__ = [
"AutoencoderKLQwenImage21",
"QwenImage21Pipeline",
"QwenImage21Transformer2DModel",
"get_qwen_image_21_post_process_func",
"get_qwen_image_21_pre_process_func",
]
def __getattr__(name: str) -> Any:
if name == "AutoencoderKLQwenImage21":
from .autoencoder_kl_qwenimage21 import AutoencoderKLQwenImage21
return AutoencoderKLQwenImage21
if name == "QwenImage21Transformer2DModel":
from .qwen_image_21_transformer import QwenImage21Transformer2DModel
return QwenImage21Transformer2DModel
if name in (
"QwenImage21Pipeline",
"get_qwen_image_21_post_process_func",
"get_qwen_image_21_pre_process_func",
):
from .pipeline_qwen_image_21 import (
QwenImage21Pipeline,
get_qwen_image_21_post_process_func,
get_qwen_image_21_pre_process_func,
)
return {
"QwenImage21Pipeline": QwenImage21Pipeline,
"get_qwen_image_21_post_process_func": get_qwen_image_21_post_process_func,
"get_qwen_image_21_pre_process_func": get_qwen_image_21_pre_process_func,
}[name]
raise AttributeError(name)
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,190 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CFG Parallel Mixin for Qwen-Image 2.1.
Differences from the 2.0 mixin (``qwen_image/cfg_parallel.py``):
- The transformer owns a KV cache for the text/condition-image prefix, so the
denoise loop passes a fresh per-generation ``kv_cache`` and names the CFG
branch (``cache_branch="cond"/"uncond"``) on every forward.
- The joint sequence is ``[condition images, ..., target image]`` — the target
noise prediction is the *tail* of the output, not the head.
"""
import logging
from typing import Any
import torch
from vllm_omni.diffusion.distributed.cfg_parallel import (
CFGParallelMixin,
_get_cfg_world_size_or_one,
_unwrap,
_wrap,
)
from vllm_omni.diffusion.distributed.parallel_state import (
get_cfg_group,
get_classifier_free_guidance_rank,
)
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
logger = logging.getLogger(__name__)
class QwenImage21CFGParallelMixin(CFGParallelMixin, ProgressBarMixin):
"""Base Mixin class for the Qwen-Image 2.1 pipeline providing shared CFG methods."""
def predict_noise_maybe_with_cfg(
self,
do_true_cfg: bool,
true_cfg_scale: float,
positive_kwargs: dict[str, Any],
negative_kwargs: dict[str, Any] | None,
cfg_normalize: bool = False,
output_slice: int | None = None,
kwargs: dict[str, Any] | None = None,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
# The base implementation slices `[:, :output_slice]` per branch, which
# matches the 2.0 condition-last layout. 2.1 concatenates condition
# latents *before* the target, so the target prediction is the tail of
# the joint output. Each branch must be tail-sliced *before* the CFG
# combine: positive and negative prompts tokenize to different lengths,
# so the full joint outputs do not align positionally.
def tail_slice(pred: torch.Tensor | tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, ...]:
if output_slice is None:
return _wrap(pred)
return tuple(p[:, -output_slice:] for p in _wrap(pred))
if do_true_cfg:
if _get_cfg_world_size_or_one() > 1:
cfg_group = get_cfg_group()
cfg_rank = get_classifier_free_guidance_rank()
branch_kwargs = positive_kwargs if cfg_rank == 0 else negative_kwargs
local_pred = tail_slice(self.predict_noise(**branch_kwargs))
gathered = [cfg_group.all_gather(p, separate_tensors=True) for p in local_pred]
positive_noise_pred = tuple(g[0] for g in gathered)
negative_noise_pred = tuple(g[1] for g in gathered)
else:
positive_noise_pred = tail_slice(self.predict_noise(**positive_kwargs))
negative_noise_pred = tail_slice(self.predict_noise(**negative_kwargs))
return self.combine_cfg_noise(
positive_noise_pred,
negative_noise_pred,
true_cfg_scale,
cfg_normalize,
**({} if kwargs is None else {"kwargs": kwargs}),
)
pred = self.predict_noise(**positive_kwargs)
if output_slice is not None:
pred = _unwrap(tail_slice(pred))
return pred
def diffuse(
self,
prompt_embeds: torch.Tensor,
prompt_embeds_mask: torch.Tensor,
negative_prompt_embeds: torch.Tensor | None,
negative_prompt_embeds_mask: torch.Tensor | None,
img_mask: torch.Tensor,
negative_img_mask: torch.Tensor | None,
latents: torch.Tensor,
image_latents: torch.Tensor | None,
img_shapes: list,
timesteps: torch.Tensor,
do_true_cfg: bool,
true_cfg_scale: float,
attention_kwargs: dict[str, Any] | None = None,
additional_transformer_kwargs: dict[str, Any] | None = None,
) -> torch.Tensor:
"""
Diffusion loop with optional classifier-free guidance.
Args:
prompt_embeds: Positive prompt embeddings
prompt_embeds_mask: Mask for positive prompt
negative_prompt_embeds: Negative prompt embeddings
negative_prompt_embeds_mask: Mask for negative prompt
img_mask: Joint-sequence image-slot mask for the positive branch
(VLM image slots plus one slot per 2x2 group of target tokens)
negative_img_mask: Same for the negative branch
latents: Noise latents to denoise (target image only, packed)
image_latents: Packed condition-image latents, prepended to the
noise latents (default: None for pure text-to-image)
img_shapes: Per-sample list of (frame, height, width) latent-token
shapes, condition images first and the target last
timesteps: Diffusion timesteps
do_true_cfg: Whether to apply CFG
true_cfg_scale: CFG scale factor
attention_kwargs: Passed through to the transformer (attn_path hint)
additional_transformer_kwargs: Extra kwargs for the transformer
Returns:
Denoised latents
"""
# Fresh cache for every generation: the pipeline is a singleton shared
# across requests, so cache state must never leak between them.
cache_enabled = getattr(self.transformer, "causal_condition", False)
kv_cache = [{} for _ in range(len(self.transformer.transformer_blocks))] if cache_enabled else None
self.scheduler.set_begin_index(0)
self.transformer.do_true_cfg = do_true_cfg
additional_transformer_kwargs = additional_transformer_kwargs or {}
with self.progress_bar(total=len(timesteps)) as pbar:
for i, t in enumerate(timesteps):
if self.interrupt:
continue
self._current_timestep = t
# Broadcast timestep to match batch size
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
# Condition image latents go in front of the noise latents.
latent_model_input = latents
if image_latents is not None:
latent_model_input = torch.cat([image_latents, latents], dim=1)
positive_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep / 1000,
"encoder_hidden_states_mask": prompt_embeds_mask,
"encoder_hidden_states": prompt_embeds,
"img_shapes": img_shapes,
"img_mask": img_mask,
"attention_kwargs": attention_kwargs,
"kv_cache": kv_cache,
"cache_branch": "cond",
**additional_transformer_kwargs,
}
if do_true_cfg:
negative_kwargs = {
"hidden_states": latent_model_input,
"timestep": timestep / 1000,
"encoder_hidden_states_mask": negative_prompt_embeds_mask,
"encoder_hidden_states": negative_prompt_embeds,
"img_shapes": img_shapes,
"img_mask": negative_img_mask,
"attention_kwargs": attention_kwargs,
"kv_cache": kv_cache,
"cache_branch": "uncond",
**additional_transformer_kwargs,
}
else:
negative_kwargs = None
# Prefill (first step) returns the full joint sequence and
# caches the prefix; decode steps return target tokens only.
# Slicing the target tail is correct for both.
noise_pred = self.predict_noise_maybe_with_cfg(
do_true_cfg,
true_cfg_scale,
positive_kwargs,
negative_kwargs,
cfg_normalize=False,
output_slice=latents.size(1),
)
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
pbar.update()
return latents
@@ -0,0 +1,457 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CUDA Graph capture for Qwen-Image-2.1 KV-cache decode steps.
Decode (denoising steps 1..N) recomputes only the target image's tokens: the
shape is identical every step and the timestep-independent prefix K/V come from
the cross-step KV cache. That is the textbook CUDA-graph case, except the
default cache stores prefix K/V in freshly allocated tensors, so a captured
graph would bind addresses that the next request (or a step-mode cache merge)
no longer owns.
Requests retain ownership of their prefix K/V. Decode refreshes separate
fixed-address graph buffers when the active prefix tensors change. Graphs
are keyed by branch, batch, prefix length, complete image layout,
dtype, device and backend, so equal token counts with different RoPE layouts stay separate.
Each key admits a single live owner, tracked by a weak reference to the
registering request's prefix K tensor. A second in-flight request whose cache
aliases the same key is refused graph decode and stays eager until the
owner's cache is released, so shared static buffers are never overwritten
under a concurrent owner.
Memory note: graph entries own an additional copy of the prefix K/V. Decode
attention concatenates that prefix with target K/V inside the captured region;
transient tensors use the graph memory pool.
The model runner regionally torch.compile's the transformer blocks even when
decode graphs are enabled, so capture records the compiled (fused) kernels;
inductor's own cudagraphs stay off so the two graph layers never stack.
Fallbacks (all logged, all eager): CUDA unavailable, model-level CPU offload
without persistent weight staging, sequence/ring/tensor parallelism,
HSDP/offload hooks, KV-cache quantization, dynamic LoRA wrappers, padded text
masks (the masked attention path branches on mask contents, which cannot be
captured), unknown graph key at decode, a second live request aliasing an
owned graph key, and capture failure.
"""
from __future__ import annotations
import weakref
from collections import OrderedDict
from typing import TYPE_CHECKING
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm_omni.platforms import current_omni_platform
if TYPE_CHECKING:
from vllm_omni.diffusion.models.qwen_image_21.qwen_image_21_transformer import (
QwenImage21Transformer2DModel,
)
logger = init_logger(__name__)
def _dynamic_lora_wrappers_present(module: torch.nn.Module) -> bool:
"""True once ``DiffusionLoRAManager`` has wrapped layers under ``module``.
A captured graph records the layers and branches that ran at capture time;
binding or rescaling an adapter afterwards changes the math without
changing the shapes the graph was keyed on, so graphs and dynamic LoRA
must not mix. Same reasoning as ``sensenova_u1.paged_decode``.
"""
try:
from vllm.lora.layers import BaseLayerWithLoRA
except ImportError: # pragma: no cover - depends on the wheel
return False
return any(isinstance(m, BaseLayerWithLoRA) for m in module.modules())
class QwenImage21DecodeGraphEntry:
"""Static state for one captured decode graph.
Buffers are created outside inference mode so replay-side ``copy_`` is
legal regardless of the caller's grad context. ``graph`` is None until the
first decode step captures it. Failed entries are replaced by a ``None``
marker in the manager so their buffers can be released.
"""
def __init__(
self,
*,
branch: str,
batch_size: int,
prefix_len: int,
target_tokens: int,
in_channels: int,
dtype: torch.dtype,
device: torch.device,
kv_shapes: list[torch.Size],
kv_dtype: torch.dtype,
freqs: torch.Tensor,
):
self.branch = branch
self.target_tokens = target_tokens
self.prefix_len = prefix_len
with torch.inference_mode(False):
self.hidden = torch.zeros(batch_size, target_tokens, in_channels, dtype=dtype, device=device)
self.timestep = torch.zeros(batch_size, dtype=dtype, device=device)
self.k = [torch.zeros(shape, dtype=kv_dtype, device=device) for shape in kv_shapes]
self.v = [torch.zeros(shape, dtype=kv_dtype, device=device) for shape in kv_shapes]
self.target_token_mask = torch.ones(target_tokens, dtype=torch.bool, device=device)
# RoPE frequencies are layout-dependent but step-independent. Clone
# into a normal (non-inference) tensor: the prefill runs under
# inference_mode, and the captured body takes views of this tensor
# outside it, which is forbidden for inference tensors.
self.freqs = freqs.clone()
self.attn_metadata = None
# Per-block cache dicts with the same shape as the legacy protocol.
self.block_caches = [{branch: {"key": self.k[i], "value": self.v[i]}} for i in range(len(kv_shapes))]
self.graph: torch.cuda.CUDAGraph | None = None
self.output: torch.Tensor | None = None
self.captures = 0
self.prefix_sources: list[tuple[weakref.ReferenceType[torch.Tensor], int | None]] = []
def refresh_prefix(self, kv_cache: list[dict[str, dict[str, torch.Tensor]]]) -> None:
sources = [block[self.branch][name] for block in kv_cache for name in ("key", "value")]
versions = [None if tensor.is_inference() else tensor._version for tensor in sources]
if len(sources) == len(self.prefix_sources) and all(
previous() is tensor and version == current_version
for (previous, version), tensor, current_version in zip(self.prefix_sources, sources, versions)
):
return
# Prefill replaces inference-mode tensors; decode treats them as immutable.
# Weak references avoid retaining completed requests or mistaking a reused
# allocation for the same prefix. Normal tensors also track in-place writes.
self.prefix_sources = []
for i, block in enumerate(kv_cache):
self.k[i].copy_(block[self.branch]["key"])
self.v[i].copy_(block[self.branch]["value"])
self.prefix_sources = [(weakref.ref(tensor), version) for tensor, version in zip(sources, versions)]
def capture(self, body) -> None:
"""Warm up on a side stream, then capture one decode step.
Warm-up runs the same computation it will record, allocating cuBLAS
workspaces and any lazy buffers outside the capture. Decode writes
nothing to the cache (prefix K/V are read-only), so warm-up is
idempotent.
"""
side = torch.cuda.Stream()
side.wait_stream(torch.cuda.current_stream())
try:
with torch.cuda.stream(side), torch.inference_mode(False), torch.no_grad():
for _ in range(2):
body()
finally:
torch.cuda.current_stream().wait_stream(side)
graph = torch.cuda.CUDAGraph()
# The platform-wide pool is shared with every other captured path in
# the tree; entries only keep their *output* alive, whose tiny size
# makes reuse safe because the caller clones it before any other graph
# on the pool replays.
with torch.inference_mode(False), torch.no_grad():
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
self.output = body()
self.graph = graph
self.captures += 1
class QwenImage21DecodeGraphManager:
"""Owns the static KV buffers and captured graphs of one transformer."""
def __init__(self, model: QwenImage21Transformer2DModel, max_entries: int = 8, model_level_offload: bool = False):
self.model = model
self.max_entries = max_entries
self.model_level_offload = model_level_offload
self.entries: OrderedDict[tuple, QwenImage21DecodeGraphEntry | None] = OrderedDict()
# Key -> weak reference to the prefix K tensor of the request that owns
# the entry's static buffers. Dead references mean the owning request
# finished and its cache was released, so the key is free to adopt.
self._owners: dict[tuple, weakref.ReferenceType[torch.Tensor]] = {}
self._static_eligible: bool | None = None
# ── eligibility ──
def _offload_reason(self) -> str | None:
"""Reason decode graphs are incompatible with model-level offload, if any.
Model-level (sequential) offload registers its hook on the top-level
transformer module, not on individual blocks. With persistent staging
the hook keeps the DiT on fixed device storage and only rebinds
``p.data`` around each generation, and the swap completes in
``pre_forward`` before capture or replay begin, so captured weight
pointers stay valid. Without staging every swap allocates fresh
storage and replay would read stale pointers, so capture stays
disabled and decode falls back to eager.
"""
registry = getattr(self.model, "_hook_registry", None)
hooks = registry._hooks if registry is not None else {}
if hooks:
from vllm_omni.diffusion.offloader.sequential_backend import (
SequentialOffloadHook,
sequential_offload_staging_active,
)
if set(hooks) <= {SequentialOffloadHook._HOOK_NAME} and sequential_offload_staging_active(self.model):
return None
return "model module carries offload/cache hooks"
if self.model_level_offload:
return "model-level CPU offload without persistent staging swaps weight storage after capture"
return None
def _check_static_eligibility(self) -> bool:
"""One-time checks that cannot change after weights are loaded."""
model = self.model
reason = self._offload_reason()
param = next(model.parameters(), None)
if reason is None and (param is None or param.device.type != "cuda"):
reason = f"model is not on CUDA (device={None if param is None else param.device})"
elif reason is None:
parallel_config = getattr(model, "parallel_config", None)
if parallel_config is not None:
sp = getattr(parallel_config, "sequence_parallel_size", None) or 1
if sp > 1 or getattr(parallel_config, "ring_degree", 1) > 1:
reason = "sequence/ring parallelism changes decode shapes per rank"
elif getattr(parallel_config, "use_hsdp", False):
reason = "HSDP wraps blocks with gather hooks outside the captured region"
if reason is None:
from vllm.distributed import get_tensor_model_parallel_world_size
if get_tensor_model_parallel_world_size() > 1:
reason = "tensor parallelism runs collectives inside the captured region"
if reason is None:
for block in model.transformer_blocks:
if hasattr(block, "_hook_registry") or hasattr(block, "_omni_original_forward"):
reason = "transformer blocks carry offload/cache hooks"
break
if reason is None and model.transformer_blocks:
attn_layer = model.transformer_blocks[0].attn.attn
if getattr(attn_layer, "_kv_cache_dtype", None) is not None:
reason = "KV-cache quantization is enabled"
if reason is not None:
logger.warning_once("Qwen-Image-2.1 CUDA graph decode disabled: %s. Falling back to eager decode.", reason)
return False
return True
def eligible(self) -> bool:
if self._static_eligible is None:
self._static_eligible = self._check_static_eligibility()
return self._static_eligible
def _backend_name(self) -> str:
backend = self.model.transformer_blocks[0].attn.attn.attn_backend
return backend.get_name() if backend is not None else "custom"
# ── prefill registration ──
def register_prefill(
self,
*,
kv_cache: list[dict[str, dict[str, torch.Tensor]]],
cache_branch: str,
prefix_len: int,
img_shapes: list[tuple[int, int, int]],
target_freqs: torch.Tensor,
joint_key_valid: torch.Tensor | None,
dtype: torch.dtype,
) -> None:
"""Prepare graph buffers without changing the request's prefix K/V.
The first in-flight request to register a key owns its static buffers;
registrations for the same key from another live request are refused
so their decode stays eager until the owner completes.
"""
if not self.eligible() or torch.compiler.is_compiling():
return
if _dynamic_lora_wrappers_present(self.model):
if self.entries:
logger.warning_once(
"Qwen-Image-2.1 CUDA graph decode: LoRA wrappers appeared; "
"dropping captured graphs and falling back to eager decode."
)
self.entries.clear()
return
if joint_key_valid is not None and not bool(joint_key_valid.all()):
# The masked decode path (varlen-unpack or 4D mask) branches on mask
# contents and cannot be captured; this request stays eager.
logger.warning_once(
"Qwen-Image-2.1 CUDA graph decode: padded text mask present; this request falls back to eager decode."
)
return
first = kv_cache[0].get(cache_branch)
if first is None or "key" not in first:
return
if any(name.endswith("_scale") for name in first):
# Quantized (FP8) prefix storage: the static K/V buffers assume a single
# dtype and the captured body would not see the dequant scales (a bf16 K +
# fp8 V mix would even copy_ fp8 into bf16 buffers, silently dropping the
# scales). Keep quantized-cache requests on the eager decode path.
logger.warning_once(
"Qwen-Image-2.1 CUDA graph decode: quantized prefix KV cache "
"(prefix_kv_cache_dtype) is not graph-capturable; falling back to eager decode."
)
return
batch_size = first["key"].shape[0]
key = (
cache_branch,
batch_size,
prefix_len,
tuple(tuple(shape) for shape in img_shapes),
dtype,
first["key"].device.index,
self._backend_name(),
)
if key not in self.entries:
while len(self.entries) >= self.max_entries:
evicted_key, evicted_entry = self.entries.popitem(last=False)
self._owners.pop(evicted_key, None)
del evicted_entry
logger.debug("Evicting decode graph entry %s (max_entries=%d)", evicted_key, self.max_entries)
allocation_error = None
try:
self.entries[key] = QwenImage21DecodeGraphEntry(
branch=cache_branch,
batch_size=batch_size,
prefix_len=prefix_len,
target_tokens=target_freqs.shape[0],
in_channels=self.model.in_channels,
dtype=dtype,
device=first["key"].device,
kv_shapes=[block_cache[cache_branch]["key"].shape for block_cache in kv_cache],
kv_dtype=first["key"].dtype,
freqs=target_freqs,
)
except torch.OutOfMemoryError as exc:
allocation_error = str(exc)
if allocation_error is not None:
# Keep a bounded failure marker without retaining tensors or the exception traceback.
self.entries[key] = None
current_omni_platform.empty_cache()
logger.warning(
"Qwen-Image-2.1 CUDA graph allocation failed for key=%s: %s. Falling back to eager decode.",
key,
allocation_error,
)
return
else:
self.entries.move_to_end(key)
entry = self.entries[key]
if entry is not None:
owner = self._owners.get(key)
owner_prefix = owner() if owner is not None else None
if owner_prefix is not None and owner_prefix is not first["key"]:
# Another in-flight request still owns this entry's static
# buffers; refreshing them would overwrite that request's
# prefix mid-generation. This request decodes eagerly until
# the owner's cache is released and the weak reference dies.
logger.warning_once(
"Qwen-Image-2.1 CUDA graph decode: entry key=%s is owned by another in-flight "
"request; this request falls back to eager decode.",
key,
)
return
self._owners[key] = weakref.ref(first["key"])
entry.prefix_sources = []
logger.debug(
"Registered decode graph entry key=%s (entries=%d, captured=%d)",
key,
len(self.entries),
sum(1 for e in self.entries.values() if e is not None and e.graph is not None),
)
# ── decode replay ──
def try_decode(
self,
*,
hidden_states: torch.Tensor,
timestep: torch.Tensor,
kv_cache: list[dict[str, dict[str, torch.Tensor]]],
cache_branch: str,
img_shapes: list[tuple[int, int, int]],
img_mask: torch.Tensor,
encoder_hidden_states_mask: torch.Tensor | None,
) -> torch.Tensor | None:
"""Replay the captured decode step, or return None to fall back to eager."""
if not self.eligible() or torch.compiler.is_compiling():
return None
if _dynamic_lora_wrappers_present(self.model):
self.clear()
return None
if encoder_hidden_states_mask is not None:
text_positions = ~img_mask[0, : encoder_hidden_states_mask.shape[1]]
if not bool(encoder_hidden_states_mask[:, text_positions].all()):
return None
first = kv_cache[0][cache_branch]
cached_key = first["key"]
if cached_key.device.type != "cuda":
# Registration is device-agnostic (unit tests exercise the buffer
# machinery on CPU), but capture/replay is CUDA-only: an empty
# graph over CPU ops would replay as a stale no-op.
return None
key = (
cache_branch,
cached_key.shape[0],
cached_key.shape[1],
tuple(tuple(shape) for shape in img_shapes),
hidden_states.dtype,
cached_key.device.index,
self._backend_name(),
)
entry = self.entries.get(key)
if entry is None:
return None
self.entries.move_to_end(key)
owner = self._owners.get(key)
owner_prefix = owner() if owner is not None else None
if owner_prefix is not None and owner_prefix is not cached_key:
# The entry is owned by a different in-flight request (see
# register_prefill); this request stays on the eager decode path
# until the owner's cache is released.
return None
entry.hidden.copy_(hidden_states[:, -entry.target_tokens :])
entry.timestep.copy_(timestep)
# Graph buffers are scratch space; request caches must never alias them.
entry.refresh_prefix(kv_cache)
if entry.graph is None:
capture_error = None
try:
entry.capture(lambda: self.model._decode_graph_forward(entry))
except Exception as exc:
capture_error = str(exc)
if capture_error is not None:
self.entries[key] = None
del entry
current_omni_platform.empty_cache()
logger.warning(
"Qwen-Image-2.1 CUDA graph capture failed for key=%s: %s. "
"Falling back to eager decode for this shape.",
key,
capture_error,
)
return None
logger.info("Captured Qwen-Image-2.1 decode graph for key=%s", key)
entry.graph.replay()
assert entry.output is not None
# Clone before returning: the output lives in the shared graph pool and
# the next replay (of this or another graph on the pool) overwrites it.
return entry.output.clone()
def clear(self) -> None:
"""Drop all entries, freeing buffers and captured graphs."""
self.entries.clear()
self._owners.clear()
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -98,6 +98,12 @@ class PinnedModuleStager:
A module iterable is treated as one staging group. It uses one copy stream
and one reusable completion event. Tensors sharing storage keep their
shapes, strides, offsets, dtypes, and aliases across every transition.
With ``retain_device_storage`` the device storages survive ``offload``:
``load`` then only rebinds to the same fixed storages (weights are
immutable, so no H2D copy repeats). The stable storage addresses keep
CUDA-graph-captured weight pointers valid across offload/load swaps, at
the cost of keeping the device storage resident while offloaded.
"""
def __init__(
@@ -108,6 +114,7 @@ class PinnedModuleStager:
pin_memory: bool = True,
copy_stream: Any | None = None,
cache_retention: BoundedAllocatorCache | None = None,
retain_device_storage: bool = False,
) -> None:
modules = (module,) if isinstance(module, nn.Module) else tuple(module)
if not modules or not all(isinstance(item, nn.Module) for item in modules):
@@ -117,6 +124,7 @@ class PinnedModuleStager:
self.copy_stream = copy_stream if copy_stream is not None else current_omni_platform.Stream()
self._ready_event = current_omni_platform.Event()
self.cache_retention = cache_retention
self.retain_device_storage = retain_device_storage
self.loaded = False
self._groups = self._snapshot_groups(modules, pin_memory=pin_memory)
self._device_storages: list[torch.Tensor] = []
@@ -234,6 +242,12 @@ class PinnedModuleStager:
def load(self) -> None:
if self.loaded:
return
if self._device_storages:
# Storage retained from a previous load still holds the immutable
# weights; rebinding suffices, no H2D copy repeats.
self._bind(self._device_storages)
self.loaded = True
return
try:
self._load_once()
except torch.OutOfMemoryError:
@@ -263,7 +277,8 @@ class PinnedModuleStager:
self.loaded = False
self._release_cache(force=True)
raise
self._device_storages.clear()
if not self.retain_device_storage:
self._device_storages.clear()
self.loaded = False
self._release_cache()
@@ -13,6 +13,7 @@ from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
from vllm_omni.platforms import current_omni_platform
from .base import OffloadBackend, OffloadConfig, SupportsModelCpuOffload
from .module_residency import PinnedModuleStager
from .plan_resolver import resolve_offload_plan
logger = init_logger(__name__)
@@ -60,6 +61,7 @@ class SequentialOffloadHook(ModelHook):
pin_memory: bool = True,
use_hsdp: bool = False,
offload_after_context: bool = True,
persistent_staging: bool = False,
):
# Modules to offload to CPU before this module runs
self.offload_targets = offload_targets
@@ -67,6 +69,21 @@ class SequentialOffloadHook(ModelHook):
self.pin_memory = pin_memory
self.use_hsdp = use_hsdp
self.offload_after_context = offload_after_context
self.persistent_staging = persistent_staging
self._stager: PinnedModuleStager | None = None
def initialize_hook(self, module: nn.Module) -> nn.Module:
if self.persistent_staging and not self.use_hsdp:
# Fixed staging storage keeps CUDA-graph-captured weight pointers
# valid across offload/load swaps. HSDP keeps the plain move
# semantics (decode graphs refuse HSDP anyway).
self._stager = PinnedModuleStager(
module,
self.device,
pin_memory=self.pin_memory,
retain_device_storage=True,
)
return module
@staticmethod
def _move_params(
@@ -101,7 +118,23 @@ class SequentialOffloadHook(ModelHook):
moved = True
return moved
@staticmethod
def _module_stager(module: nn.Module) -> PinnedModuleStager | None:
"""The persistent stager owned by ``module``'s own sequential hook, if any.
Cross-module swaps call ``_to_cpu(target)`` through the *running*
module's hook, so staging state must be resolved on the target's hook
to keep its ``loaded`` bookkeeping consistent.
"""
registry: HookRegistry | None = getattr(module, "_hook_registry", None)
hook = registry.get_hook(SequentialOffloadHook._HOOK_NAME) if registry is not None else None
return hook._stager if isinstance(hook, SequentialOffloadHook) else None
def _to_cpu(self, module: nn.Module) -> None:
stager = self._module_stager(module)
if stager is not None:
stager.offload()
return
# XPU's allocator doesn't respect stream dependencies in empty_cache,
# so non-blocking copies can race with cache eviction. Use blocking
# copies on XPU to avoid NULL pointer errors during DMA.
@@ -116,6 +149,10 @@ class SequentialOffloadHook(ModelHook):
current_omni_platform.empty_cache()
def _to_gpu(self, module: nn.Module) -> None:
stager = self._module_stager(module)
if stager is not None:
stager.load()
return
self._move_params(module, self.device, non_blocking=False)
def pre_forward(self, module: nn.Module, *args, **kwargs) -> tuple[tuple, dict]:
@@ -147,6 +184,7 @@ def apply_sequential_offload(
offload_initial_dits: bool = False,
offload_dit_modules: Collection[nn.Module] | None = None,
offload_encoder_modules: Collection[nn.Module] | None = None,
persistent_dit_staging: bool = False,
) -> None:
"""Apply sequential offloading hooks to DiT and encoder modules.
@@ -165,6 +203,10 @@ def apply_sequential_offload(
every DiT for backward compatibility.
offload_encoder_modules: Encoder/stage modules allowed to move to CPU.
None selects every supplied module for backward compatibility.
persistent_dit_staging: Keep DiT weights on fixed device staging
storage across swaps, so CUDA-graph-captured weight pointers stay
valid. Encoders keep plain move semantics so their memory is
actually freed while the DiT runs.
Example:
>>> apply_sequential_offload(
@@ -193,6 +235,7 @@ def apply_sequential_offload(
pin_memory=pin_memory,
use_hsdp=use_hsdp,
offload_after_context=id(dit_mod) in selected_dit_ids,
persistent_staging=persistent_dit_staging,
)
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
logger.debug("Registered offload hook for %s", dit_mod.__class__.__name__)
@@ -261,6 +304,15 @@ def _get_sequential_offload_hook(module: nn.Module) -> SequentialOffloadHook:
return hook
def sequential_offload_staging_active(module: nn.Module) -> bool:
"""True when the module's sequential offload hook serves fixed device storage.
Fixed staging storage keeps CUDA-graph-captured weight pointers valid
across offload/load swaps.
"""
return SequentialOffloadHook._module_stager(module) is not None
@contextmanager
def sequential_offload_component(module: nn.Module) -> Iterator[None]:
"""Activate and release a hooked component called outside ``forward``."""
@@ -362,6 +414,14 @@ class ModelLevelOffloadBackend(OffloadBackend):
use_hsdp=self.config.use_hsdp,
offload_dit_modules=selected_dits,
offload_encoder_modules=selected_encoders,
# Fixed DiT staging keeps decode-graph weight pointers valid;
# only models that capture weight pointers (Qwen-Image-2.1's
# CUDA-graph decode) pay for it — every other model keeps
# plain move semantics so offloading actually frees DiT VRAM
# while the encoders run.
persistent_dit_staging=(
self.device.type == "cuda" and any(getattr(dit, "enable_cuda_graph_decode", False) for dit in dits)
),
)
except BaseException:
try:
+11
View File
@@ -56,6 +56,11 @@ _DIFFUSION_MODELS = {
"pipeline_qwen_image_layered",
"QwenImageLayeredPipeline",
),
"QwenImage21Pipeline": (
"qwen_image_21",
"pipeline_qwen_image_21",
"QwenImage21Pipeline",
),
"GlmImagePipeline": (
"glm_image",
"pipeline_glm_image",
@@ -426,6 +431,10 @@ _NO_CACHE_ACCELERATION = {
"Pi0Pipeline",
"Pi05Pipeline",
"LingBotWorldCausalDMDPipeline",
# Qwen-Image 2.1's transformer carries its own prefix KV cache (keyed by CFG
# branch) across denoising steps, which conflicts with cache_dit / tea_cache
# step-skipping hooks.
"QwenImage21Pipeline",
}
@@ -651,6 +660,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = {
"AnimaPipeline": "get_anima_post_process_func",
"QwenImageEditPipeline": "get_qwen_image_edit_post_process_func",
"QwenImageEditPlusPipeline": "get_qwen_image_edit_plus_post_process_func",
"QwenImage21Pipeline": "get_qwen_image_21_post_process_func",
"GlmImagePipeline": "get_glm_image_post_process_func",
"ZImagePipeline": "get_post_process_func",
"OvisImagePipeline": "get_ovis_image_post_process_func",
@@ -740,6 +750,7 @@ _DIFFUSION_PRE_PROCESS_FUNCS = {
"BooguImageTurboPipeline": "get_boogu_image_pre_process_func",
"QwenImageEditPipeline": "get_qwen_image_edit_pre_process_func",
"QwenImageEditPlusPipeline": "get_qwen_image_edit_plus_pre_process_func",
"QwenImage21Pipeline": "get_qwen_image_21_pre_process_func",
"LongCatImageEditPipeline": "get_longcat_image_edit_pre_process_func",
"LongCatVideoAvatarPipeline": "get_longcat_video_avatar_pre_process_func",
"QwenImageLayeredPipeline": "get_qwen_image_layered_pre_process_func",
+2
View File
@@ -76,6 +76,8 @@ class OmniDiffusionRequest:
# A model preprocessor may keep selected requests on the legacy full-forward
# path even when the engine is globally configured for step execution.
use_step_execution: bool = True
# Whether first and later denoising steps may share a step batch.
allow_mixed_step_phases: bool = True
# KV-recv wall-clock (ms), set by the runner's _prepare_request_for_forward
# and carried to DiffusionOutput for the vllm_omni:diffusion_kv_load_s metric.
kv_recv_ms: float = 0.0
@@ -13,6 +13,7 @@ from vllm_omni.diffusion.sched.base_scheduler import BaseScheduler
from vllm_omni.diffusion.sched.interface import (
DiffusionRequestStatus,
DiffusionSchedulerOutput,
SchedulerRequestState,
)
if TYPE_CHECKING:
@@ -62,6 +63,19 @@ class StepScheduler(BaseScheduler):
)
return request_id
def _can_schedule_waiting(self, state: SchedulerRequestState) -> bool:
if not super()._can_schedule_waiting(state):
return False
is_first_step = self._request_progress[state.request_id].current_step == 0
for request_id in self._running:
running = self._request_states[request_id]
if state.req.allow_mixed_step_phases and running.req.allow_mixed_step_phases:
continue
if (self._request_progress[request_id].current_step == 0) != is_first_step:
return False
return True
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: BaseRunnerOutput) -> set[str]:
scheduled_request_ids = sched_output.scheduled_request_ids
if not scheduled_request_ids and not sched_output.finished_req_ids:
@@ -206,7 +206,7 @@ class DiffusionModelRunner(DiffusionStagePayloadMixin):
self.kv_transfer_manager = (
payload_transfer_manager if getattr(od_config, "kv_transfer_config", None) is None else None
)
self.init_omni_connectors(od_config, payload_transfer_manager, synchronous=True)
self.init_omni_connectors(od_config, payload_transfer_manager, synchronous=True) # type: ignore[arg-type]
self._kv_connector = None
from vllm_omni.diffusion.diffusion_kv.kv_connector import KVReceiveProgress, native_prefetch_enabled
@@ -250,14 +250,25 @@ class DiffusionModelRunner(DiffusionStagePayloadMixin):
if model is None:
return
compile_kwargs: dict[str, Any] = {"dynamic": self.od_config.diffusion_compile_dynamic}
if getattr(model, "enable_cuda_graph_decode", False):
# Decode-graph models still compile their blocks: graph capture
# records the compiled (fused) kernels. Scope inductor cudagraphs
# to this compile so the two graph layers never stack, without
# changing torch.compile for every other model in the process.
compile_kwargs["options"] = {
"triton.cudagraphs": False,
"triton.cudagraph_trees": False,
}
logger.info("Model runner: %s combines CUDA graph decode with torch.compile.", attr_name)
compile_granularity = self.od_config.diffusion_compile_granularity
compile_dynamic = self.od_config.diffusion_compile_dynamic
try:
if compile_granularity == "full":
model.compile(dynamic=compile_dynamic)
model.compile(**compile_kwargs)
compiled_model = model
else:
compiled_model = regionally_compile(model, dynamic=compile_dynamic)
compiled_model = regionally_compile(model, **compile_kwargs)
setattr(self.pipeline, attr_name, compiled_model)
except Exception as e:
logger.warning(
@@ -275,7 +286,7 @@ class DiffusionModelRunner(DiffusionStagePayloadMixin):
"compilation errors may surface on the first request.",
attr_name,
compile_granularity,
compile_dynamic,
compile_kwargs["dynamic"],
)
def load_model(
@@ -1454,7 +1465,7 @@ class DiffusionModelRunner(DiffusionStagePayloadMixin):
else req.denoise_completed
)
if finished and result is not None:
self._maybe_send_stage_payload([req], [result])
self._maybe_send_stage_payload([req], [result]) # type: ignore[list-item]
runner_output_list.append(
RunnerOutput(
request_id=req.request_id,