[New model] Qwen-Image-2.1 support (rebase + review follow-ups) (#8099)
Signed-off-by: NumberWan <wantszkin2003@gmail.com> Signed-off-by: SamitHuang <285365963@qq.com> Signed-off-by: Samit <285365963@qq.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: SamitHuang <285365963@qq.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
52 files changed
+8058
-136
No files matched your search
@@ -10,11 +10,13 @@ from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionMetadata,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.backends.sdpa import _maybe_reshape_attn_mask
|
||||
from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import piecewise_attn
|
||||
|
||||
|
||||
class CuDNNAttentionBackend(AttentionBackend):
|
||||
accept_output_buffer: bool = True
|
||||
supports_prefix_kv_slicing: bool = True
|
||||
supports_piecewise_spans: bool = True
|
||||
|
||||
# cuDNN 9.5+ FMHA on Blackwell: head_dim divisible by 8 and at most 256
|
||||
# for BF16/FP16. Used by automatic platform selection; explicit CUDNN_ATTN
|
||||
@@ -74,6 +76,23 @@ class CuDNNAttentionImpl(AttentionImpl):
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata | None = None,
|
||||
) -> torch.Tensor:
|
||||
if (
|
||||
attn_metadata is not None
|
||||
and attn_metadata.full_attn_spans is not None
|
||||
and (attn_metadata.attn_mask is None or attn_metadata.attn_mask.ndim != 4)
|
||||
):
|
||||
if attn_metadata.attn_mask is not None or "valid_kv_length" in attn_metadata.extra:
|
||||
raise ValueError("Piecewise cuDNN attention cannot combine spans with a padding mask.")
|
||||
return piecewise_attn(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_metadata.full_attn_spans,
|
||||
self.softmax_scale,
|
||||
self._piecewise_attention,
|
||||
query_ranges=attn_metadata.query_ranges,
|
||||
)
|
||||
|
||||
attention_mask = None
|
||||
if attn_metadata:
|
||||
valid_kv_length = attn_metadata.extra.get("valid_kv_length")
|
||||
@@ -97,6 +116,19 @@ class CuDNNAttentionImpl(AttentionImpl):
|
||||
mask_mode="broadcast_k",
|
||||
)
|
||||
|
||||
return self._attention(query, key, value, attention_mask, self.causal, self.softmax_scale)
|
||||
|
||||
def _piecewise_attention(self, query, key, value, *, causal, softmax_scale):
|
||||
mask = None
|
||||
if causal:
|
||||
# SDPA's causal flag aligns at the top left. Text segments need
|
||||
# bottom-right alignment so every query can also see its prefix.
|
||||
q_len, kv_len = query.shape[1], key.shape[1]
|
||||
mask = torch.ones(q_len, kv_len, dtype=torch.bool, device=query.device).tril(kv_len - q_len)
|
||||
mask = mask[None, None]
|
||||
return self._attention(query, key, value, mask, False, softmax_scale)
|
||||
|
||||
def _attention(self, query, key, value, attention_mask, causal, softmax_scale):
|
||||
enable_gqa = query.shape[2] != key.shape[2]
|
||||
kv_seq_len = key.shape[1]
|
||||
query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value))
|
||||
@@ -130,8 +162,8 @@ class CuDNNAttentionImpl(AttentionImpl):
|
||||
value,
|
||||
attn_mask=attention_mask,
|
||||
dropout_p=0.0,
|
||||
is_causal=self.causal,
|
||||
scale=self.softmax_scale,
|
||||
is_causal=causal,
|
||||
scale=softmax_scale,
|
||||
enable_gqa=enable_gqa,
|
||||
)
|
||||
return output.permute(0, 2, 1, 3)
|
||||
@@ -16,7 +16,7 @@ import diffusers
|
||||
import huggingface_hub
|
||||
import torch
|
||||
from PIL import Image
|
||||
from pydantic import Field, model_validator
|
||||
from pydantic import model_validator
|
||||
from typing_extensions import Self
|
||||
from vllm.config.utils import config
|
||||
from vllm.logger import init_logger
|
||||
@@ -981,6 +981,19 @@ class OmniDiffusionConfig:
|
||||
|
||||
# Compilation
|
||||
enforce_eager: bool = False
|
||||
# Capture fixed-shape KV-cache decode (denoising) steps into CUDA graphs.
|
||||
# Currently only Qwen-Image-2.1's transformer implements this; other models
|
||||
# ignore the flag. This is a per-model diffusion path, independent of the
|
||||
# AR engine's ``compilation_config.cudagraph_mode`` (which does not apply
|
||||
# to diffusion stages). It stacks with ``diffusion_compile_granularity``:
|
||||
# the DiT blocks are still torch.compile'd and graph capture records the
|
||||
# compiled (fused) kernels, while inductor's own cudagraphs stay off.
|
||||
# ``enforce_eager=True`` forces eager decode and
|
||||
# disables capture regardless of this flag; unsupported configurations
|
||||
# (SP/TP, ring, HSDP, dynamic LoRA, padded masks, quantized prefix KV, or
|
||||
# a second in-flight request aliasing the same graph key) log and fall
|
||||
# back to eager decode.
|
||||
enable_cuda_graph_decode: bool = True
|
||||
# Controls the generic compilation path used when a pipeline does not
|
||||
# provide its own setup_compile() implementation.
|
||||
diffusion_compile_granularity: str = "regional"
|
||||
@@ -1140,7 +1153,7 @@ class OmniDiffusionConfig:
|
||||
request_batch_max_wait_ms: float = 0.0
|
||||
|
||||
# Supplementary model specific parameters
|
||||
extras: dict[str, Any] = Field(default_factory=dict)
|
||||
extras: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def is_moe(self) -> bool:
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from diffusers.models.autoencoders.vae import DecoderOutput
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
|
||||
DistributedOperator,
|
||||
DistributedVaeMixin,
|
||||
GridSpec,
|
||||
TileTask,
|
||||
)
|
||||
from vllm_omni.diffusion.models.qwen_image_21.autoencoder_kl_qwenimage21 import (
|
||||
AutoencoderKLQwenImage21,
|
||||
_cudnn_deterministic,
|
||||
_unpatchify,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class DistributedAutoencoderKLQwenImage21(AutoencoderKLQwenImage21, DistributedVaeMixin):
|
||||
@classmethod
|
||||
def from_pretrained(cls, *args: Any, **kwargs: Any):
|
||||
model = super().from_pretrained(*args, **kwargs)
|
||||
model.init_distributed()
|
||||
return model
|
||||
|
||||
def tile_split(self, z: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
|
||||
# mostly copy from AutoencoderKLQwenImage21.tiled_decode
|
||||
_, _, num_frames, height, width = z.shape
|
||||
sample_height = height * self.spatial_compression_ratio
|
||||
sample_width = width * self.spatial_compression_ratio
|
||||
|
||||
tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio
|
||||
tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio
|
||||
tile_latent_stride_height = self.tile_sample_stride_height // self.spatial_compression_ratio
|
||||
tile_latent_stride_width = self.tile_sample_stride_width // self.spatial_compression_ratio
|
||||
|
||||
tile_sample_stride_height = self.tile_sample_stride_height
|
||||
tile_sample_stride_width = self.tile_sample_stride_width
|
||||
if self.config.patch_size is not None:
|
||||
sample_height = sample_height // self.config.patch_size
|
||||
sample_width = sample_width // self.config.patch_size
|
||||
tile_sample_stride_height = tile_sample_stride_height // self.config.patch_size
|
||||
tile_sample_stride_width = tile_sample_stride_width // self.config.patch_size
|
||||
blend_height = self.tile_sample_min_height // self.config.patch_size - tile_sample_stride_height
|
||||
blend_width = self.tile_sample_min_width // self.config.patch_size - tile_sample_stride_width
|
||||
else:
|
||||
blend_height = self.tile_sample_min_height - tile_sample_stride_height
|
||||
blend_width = self.tile_sample_min_width - tile_sample_stride_width
|
||||
|
||||
# Split z into overlapping tiles and decode them separately.
|
||||
# The tiles have an overlap to avoid seams between tiles.
|
||||
tiletask_list: list[TileTask] = []
|
||||
for i in range(0, height, tile_latent_stride_height):
|
||||
for j in range(0, width, tile_latent_stride_width):
|
||||
time_list = []
|
||||
for k in range(num_frames):
|
||||
tile = z[:, :, k : k + 1, i : i + tile_latent_min_height, j : j + tile_latent_min_width]
|
||||
time_list.append(tile)
|
||||
tiletask_list.append(
|
||||
TileTask(
|
||||
len(tiletask_list),
|
||||
(i // tile_latent_stride_height, j // tile_latent_stride_width),
|
||||
time_list,
|
||||
workload=time_list[0].shape[3] * time_list[0].shape[4],
|
||||
)
|
||||
)
|
||||
tile_spec = {
|
||||
"sample_height": sample_height,
|
||||
"sample_width": sample_width,
|
||||
"blend_height": blend_height,
|
||||
"blend_width": blend_width,
|
||||
"tile_sample_stride_height": tile_sample_stride_height,
|
||||
"tile_sample_stride_width": tile_sample_stride_width,
|
||||
}
|
||||
grid_spec = GridSpec(
|
||||
split_dims=(3, 4),
|
||||
grid_shape=(tiletask_list[-1].grid_coord[0] + 1, tiletask_list[-1].grid_coord[1] + 1),
|
||||
tile_spec=tile_spec,
|
||||
output_dtype=self.dtype,
|
||||
)
|
||||
return tiletask_list, grid_spec
|
||||
|
||||
def tile_exec(self, task: TileTask) -> torch.Tensor:
|
||||
"""Decode a single latent tile into RGB space."""
|
||||
self.clear_cache()
|
||||
time = []
|
||||
with _cudnn_deterministic():
|
||||
for k in range(len(task.tensor)):
|
||||
self._conv_idx = [0]
|
||||
tile = self.post_quant_conv(task.tensor[k])
|
||||
decoded = self.decoder(tile, feat_cache=self._feat_map, feat_idx=self._conv_idx, first_chunk=(k == 0))
|
||||
time.append(decoded)
|
||||
result = torch.cat(time, dim=2)
|
||||
return result
|
||||
|
||||
def tile_merge(self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid_spec: GridSpec) -> torch.Tensor:
|
||||
"""Merge decoded tiles into a full image."""
|
||||
grid_h, grid_w = grid_spec.grid_shape
|
||||
self.clear_cache()
|
||||
|
||||
result_rows = []
|
||||
for i in range(grid_h):
|
||||
result_row = []
|
||||
for j in range(grid_w):
|
||||
tile = coord_tensor_map[(i, j)]
|
||||
if i > 0:
|
||||
tile = self.blend_v(coord_tensor_map[(i - 1, j)], tile, grid_spec.tile_spec["blend_height"])
|
||||
if j > 0:
|
||||
tile = self.blend_h(coord_tensor_map[(i, j - 1)], tile, grid_spec.tile_spec["blend_width"])
|
||||
result_row.append(
|
||||
tile[
|
||||
:,
|
||||
:,
|
||||
:,
|
||||
: grid_spec.tile_spec["tile_sample_stride_height"],
|
||||
: grid_spec.tile_spec["tile_sample_stride_width"],
|
||||
]
|
||||
)
|
||||
result_rows.append(torch.cat(result_row, dim=-1))
|
||||
dec = torch.cat(result_rows, dim=3)[
|
||||
:, :, :, : grid_spec.tile_spec["sample_height"], : grid_spec.tile_spec["sample_width"]
|
||||
]
|
||||
|
||||
if self.config.patch_size is not None:
|
||||
dec = _unpatchify(dec, patch_size=self.config.patch_size)
|
||||
|
||||
dec = torch.clamp(dec, min=-1.0, max=1.0)
|
||||
return dec
|
||||
|
||||
def tiled_decode(self, z: torch.Tensor, return_dict: bool = True):
|
||||
if not self.is_distributed_enabled():
|
||||
return super().tiled_decode(z, return_dict=return_dict)
|
||||
|
||||
logger.debug("Decode running with distributed executor")
|
||||
result = self.distributed_executor.execute(
|
||||
z,
|
||||
DistributedOperator(split=self.tile_split, exec=self.tile_exec, merge=self.tile_merge),
|
||||
broadcast_result=True,
|
||||
)
|
||||
if not return_dict:
|
||||
return (result,)
|
||||
|
||||
return DecoderOutput(sample=result)
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import torch
|
||||
@@ -34,9 +36,9 @@ class TileTask:
|
||||
|
||||
@dataclass
|
||||
class DistributedOperator:
|
||||
split: callable
|
||||
exec: callable
|
||||
merge: callable
|
||||
split: Callable[[torch.Tensor], tuple[list[TileTask], GridSpec]]
|
||||
exec: Callable[[TileTask], torch.Tensor]
|
||||
merge: Callable[[dict[tuple[int, ...], torch.Tensor], GridSpec], torch.Tensor]
|
||||
|
||||
|
||||
class DistributedVaeExecutor:
|
||||
@@ -46,7 +48,11 @@ class DistributedVaeExecutor:
|
||||
|
||||
def __init__(self):
|
||||
# Use a dedicated process group spanning the complete worker WORLD.
|
||||
self.group = get_world_group().device_group
|
||||
world_group = get_world_group()
|
||||
self.group = world_group.device_group
|
||||
self._error_group = world_group.cpu_group
|
||||
# Reporting an OOM must not require another device allocation.
|
||||
self._error_status = torch.zeros((), dtype=torch.int32, device="cpu")
|
||||
self.world_size = dist.get_world_size(self.group)
|
||||
self.rank = dist.get_rank(self.group)
|
||||
self.parallel_size = 1
|
||||
@@ -56,8 +62,34 @@ class DistributedVaeExecutor:
|
||||
self.parallel_size = parallel_size
|
||||
self.parallel_mode = mode
|
||||
|
||||
@contextmanager
|
||||
def _sync_errors(self, stage: str):
|
||||
"""Agree on local-stage failures before entering the next collective."""
|
||||
status = 0
|
||||
message = ""
|
||||
try:
|
||||
yield
|
||||
except torch.OutOfMemoryError as error:
|
||||
status = 1
|
||||
message = str(error)
|
||||
except Exception as error:
|
||||
# A non-retryable error takes precedence over OOM on another rank.
|
||||
status = 2
|
||||
message = f"{type(error).__name__}: {error}"
|
||||
|
||||
self._error_status.fill_(status)
|
||||
dist.all_reduce(self._error_status, op=dist.ReduceOp.MAX, group=self._error_group)
|
||||
global_status = self._error_status.item()
|
||||
if global_status == 2:
|
||||
detail = message if status == 2 else "another rank failed"
|
||||
raise RuntimeError(f"Distributed VAE {stage} failed: {detail}")
|
||||
if global_status == 1:
|
||||
detail = message if status == 1 else "another rank ran out of memory"
|
||||
raise torch.OutOfMemoryError(f"Distributed VAE {stage} failed: {detail}")
|
||||
|
||||
def gather_tensors(self, tensor: torch.Tensor):
|
||||
gather_list = [torch.empty_like(tensor) for _ in range(self.world_size)]
|
||||
with self._sync_errors("gather allocation"):
|
||||
gather_list = [torch.empty_like(tensor) for _ in range(self.world_size)]
|
||||
dist.all_gather(gather_list, tensor, group=self.group)
|
||||
return gather_list if self.rank == 0 else None
|
||||
|
||||
@@ -66,11 +98,12 @@ class DistributedVaeExecutor:
|
||||
return tensor
|
||||
|
||||
def _compute_global_padding_shape(self, local_results, output_ndim: int, device):
|
||||
local_tile_max_dims = [0] * output_ndim
|
||||
for _, tile_tensor in local_results:
|
||||
for dim_idx, dim_size in enumerate(tile_tensor.shape):
|
||||
local_tile_max_dims[dim_idx] = max(local_tile_max_dims[dim_idx], dim_size)
|
||||
local_shape_stat = torch.tensor([len(local_results), *local_tile_max_dims], device=device)
|
||||
with self._sync_errors("shape preparation"):
|
||||
local_tile_max_dims = [0] * output_ndim
|
||||
for _, tile_tensor in local_results:
|
||||
for dim_idx, dim_size in enumerate(tile_tensor.shape):
|
||||
local_tile_max_dims[dim_idx] = max(local_tile_max_dims[dim_idx], dim_size)
|
||||
local_shape_stat = torch.tensor([len(local_results), *local_tile_max_dims], device=device)
|
||||
dist.all_reduce(
|
||||
local_shape_stat,
|
||||
op=dist.ReduceOp.MAX,
|
||||
@@ -100,17 +133,16 @@ class DistributedVaeExecutor:
|
||||
tid = int(meta_src[idx, 0])
|
||||
if tid < 0:
|
||||
continue
|
||||
slices = [slice(None)] * tiles_src[idx].ndim
|
||||
slices_list = [slice(None)] * tiles_src[idx].ndim
|
||||
for i, dim in enumerate(grid_spec.split_dims):
|
||||
slices[dim] = slice(0, int(meta_src[idx, i + 1]))
|
||||
slices = tuple(slices)
|
||||
coord_tensor_map[tid_coord_map[tid]] = tiles_src[idx][slices]
|
||||
slices_list[dim] = slice(0, int(meta_src[idx, i + 1]))
|
||||
coord_tensor_map[tid_coord_map[tid]] = tiles_src[idx][tuple(slices_list)]
|
||||
|
||||
return coord_tensor_map
|
||||
|
||||
def _balance_tasks(self, task_list, num_rank):
|
||||
workloads = [0] * num_rank
|
||||
assigned = [[] for _ in range(num_rank)]
|
||||
assigned: list[list[TileTask]] = [[] for _ in range(num_rank)]
|
||||
|
||||
for task in sorted(task_list, key=lambda t: t.workload, reverse=True):
|
||||
r = workloads.index(min(workloads))
|
||||
@@ -122,49 +154,51 @@ class DistributedVaeExecutor:
|
||||
def execute(self, z: torch.Tensor, operator: DistributedOperator, broadcast_result: bool = True):
|
||||
pp_size = min(self.parallel_size, self.world_size)
|
||||
|
||||
# 1. Split into tiles
|
||||
tiletask_list, grid_spec = operator.split(z)
|
||||
tid_coord_map = {task.tile_id: task.grid_coord for task in tiletask_list}
|
||||
|
||||
# 2. local decode
|
||||
assigned = self._balance_tasks(tiletask_list, pp_size)
|
||||
local_tasks = assigned[self.rank] if self.rank < pp_size else []
|
||||
local_results = [(t.tile_id, operator.exec(t)) for t in local_tasks]
|
||||
with self._sync_errors("tile decode"):
|
||||
tiletask_list, grid_spec = operator.split(z)
|
||||
tid_coord_map = {task.tile_id: task.grid_coord for task in tiletask_list}
|
||||
assigned = self._balance_tasks(tiletask_list, pp_size)
|
||||
local_tasks = assigned[self.rank] if self.rank < pp_size else []
|
||||
local_results = [(t.tile_id, operator.exec(t)) for t in local_tasks]
|
||||
|
||||
# 3. compute shape per rank
|
||||
global_padding_shape = self._compute_global_padding_shape(local_results, z.ndim, z.device)
|
||||
|
||||
# 5. prepare tile tensors
|
||||
output_dtype = grid_spec.output_dtype if grid_spec.output_dtype is not None else z.dtype
|
||||
local_tile_tensor, local_meta_tensor = self._pack_local_tiles(
|
||||
local_results, global_padding_shape, grid_spec, z.device, output_dtype
|
||||
)
|
||||
with self._sync_errors("tile packing"):
|
||||
local_tile_tensor, local_meta_tensor = self._pack_local_tiles(
|
||||
local_results, global_padding_shape, grid_spec, z.device, output_dtype
|
||||
)
|
||||
|
||||
# 6. gather tiles & meta
|
||||
meta_gather = self.gather_tensors(local_meta_tensor)
|
||||
tile_gather = self.gather_tensors(local_tile_tensor)
|
||||
|
||||
if self.rank != 0:
|
||||
result = torch.empty(0, device=z.device) # Dummy return for non-zero ranks
|
||||
else:
|
||||
# 7. reconstruct full tensor (rank 0)
|
||||
coord_tensor_map = self._unpack_tiles(meta_gather, tile_gather, grid_spec, tid_coord_map)
|
||||
result = operator.merge(coord_tensor_map, grid_spec)
|
||||
with self._sync_errors("tile merge"):
|
||||
if self.rank != 0:
|
||||
result = torch.empty(0, device=z.device) # Dummy return for non-zero ranks
|
||||
else:
|
||||
# 7. reconstruct full tensor (rank 0)
|
||||
coord_tensor_map = self._unpack_tiles(meta_gather, tile_gather, grid_spec, tid_coord_map)
|
||||
result = operator.merge(coord_tensor_map, grid_spec)
|
||||
|
||||
if broadcast_result:
|
||||
result = self._sync_final_result(result, z.ndim, z.device, output_dtype)
|
||||
return result
|
||||
|
||||
def _sync_final_result(self, rank0_result, output_ndim, output_device, output_dtype):
|
||||
shape_tensor = torch.empty((output_ndim,), device=output_device, dtype=torch.int64)
|
||||
if self.rank == 0:
|
||||
shape_tensor.copy_(torch.tensor(tuple(rank0_result.shape), device=output_device, dtype=torch.int64))
|
||||
with self._sync_errors("output shape preparation"):
|
||||
shape_tensor = torch.empty((output_ndim,), device=output_device, dtype=torch.int64)
|
||||
if self.rank == 0:
|
||||
shape_tensor.copy_(torch.tensor(tuple(rank0_result.shape), device=output_device, dtype=torch.int64))
|
||||
dist.broadcast(shape_tensor, src=0, group=self.group)
|
||||
|
||||
if self.rank != 0:
|
||||
sync_result = torch.empty(tuple(shape_tensor.tolist()), device=output_device, dtype=output_dtype)
|
||||
else:
|
||||
sync_result = rank0_result
|
||||
with self._sync_errors("output allocation"):
|
||||
if self.rank != 0:
|
||||
sync_result = torch.empty(tuple(shape_tensor.tolist()), device=output_device, dtype=output_dtype)
|
||||
else:
|
||||
sync_result = rank0_result
|
||||
dist.broadcast(sync_result, src=0, group=self.group)
|
||||
return sync_result
|
||||
|
||||
|
||||
@@ -23,6 +23,11 @@ lora_convert_mapping: dict[str, Callable] = {
|
||||
"QwenImagePipeline": _convert_non_diffusers_qwen_lora_to_diffusers,
|
||||
"QwenImageEditPipeline": _convert_non_diffusers_qwen_lora_to_diffusers,
|
||||
"QwenImageEditPlusPipeline": _convert_non_diffusers_qwen_lora_to_diffusers,
|
||||
# 2.1's single-stream blocks keep the same leaf names the converter's
|
||||
# protected n-grams cover (`attn.to_q/to_k/to_v`, `img_mlp`, ...), so the
|
||||
# 2.0 converter applies. Non-diffusers LoRAs that target the fused `to_qkv`
|
||||
# projection are not supported (the converter would split it into `to.qkv`).
|
||||
"QwenImage21Pipeline": _convert_non_diffusers_qwen_lora_to_diffusers,
|
||||
"Wan22Pipeline": _convert_non_diffusers_wan_lora_to_diffusers,
|
||||
"Wan22I2VPipeline": _convert_non_diffusers_wan_lora_to_diffusers,
|
||||
}
|
||||
|
||||
@@ -26,6 +26,8 @@ class DiffusionModelMetadata:
|
||||
# FLUX.2 Klein supports up to four reference images.
|
||||
FLUX2_KLEIN_MAX_INPUT_IMAGES = 4
|
||||
QWEN_IMAGE_EDIT_PLUS_MAX_INPUT_IMAGES = 4
|
||||
# Qwen-Image 2.1 image-conditioned generation caps condition images at 4.
|
||||
QWEN_IMAGE_21_MAX_INPUT_IMAGES = 4
|
||||
# Upstream HunyuanImage-3.0 "Multi-Image Fusion" caps reference images at 3.
|
||||
HUNYUAN_IMAGE3_MAX_INPUT_IMAGES = 3
|
||||
# Boogu-Image editing (TI2I) supports a single reference image for now.
|
||||
@@ -42,6 +44,12 @@ _DIFFUSION_MODEL_METADATA: dict[str, DiffusionModelMetadata] = {
|
||||
supports_multimodal_inputs=True,
|
||||
max_multimodal_image_inputs=QWEN_IMAGE_EDIT_PLUS_MAX_INPUT_IMAGES,
|
||||
),
|
||||
# Qwen-Image 2.1 handles both text-to-image (no image) and
|
||||
# image-conditioned requests through the same pipeline class.
|
||||
"QwenImage21Pipeline": DiffusionModelMetadata(
|
||||
supports_multimodal_inputs=True,
|
||||
max_multimodal_image_inputs=QWEN_IMAGE_21_MAX_INPUT_IMAGES,
|
||||
),
|
||||
"HunyuanImage3Pipeline": DiffusionModelMetadata(
|
||||
supports_multimodal_inputs=True,
|
||||
max_multimodal_image_inputs=HUNYUAN_IMAGE3_MAX_INPUT_IMAGES,
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Qwen Image 2.1 diffusion model components.
|
||||
|
||||
Lazy by necessity: the pipeline imports the distributed VAE
|
||||
(`vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_qwenimage21`),
|
||||
which in turn imports the plain VAE module from this package. Eager imports
|
||||
here would make that a circular import whenever the distributed module is
|
||||
imported first.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
__all__ = [
|
||||
"AutoencoderKLQwenImage21",
|
||||
"QwenImage21Pipeline",
|
||||
"QwenImage21Transformer2DModel",
|
||||
"get_qwen_image_21_post_process_func",
|
||||
"get_qwen_image_21_pre_process_func",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name == "AutoencoderKLQwenImage21":
|
||||
from .autoencoder_kl_qwenimage21 import AutoencoderKLQwenImage21
|
||||
|
||||
return AutoencoderKLQwenImage21
|
||||
if name == "QwenImage21Transformer2DModel":
|
||||
from .qwen_image_21_transformer import QwenImage21Transformer2DModel
|
||||
|
||||
return QwenImage21Transformer2DModel
|
||||
if name in (
|
||||
"QwenImage21Pipeline",
|
||||
"get_qwen_image_21_post_process_func",
|
||||
"get_qwen_image_21_pre_process_func",
|
||||
):
|
||||
from .pipeline_qwen_image_21 import (
|
||||
QwenImage21Pipeline,
|
||||
get_qwen_image_21_post_process_func,
|
||||
get_qwen_image_21_pre_process_func,
|
||||
)
|
||||
|
||||
return {
|
||||
"QwenImage21Pipeline": QwenImage21Pipeline,
|
||||
"get_qwen_image_21_post_process_func": get_qwen_image_21_post_process_func,
|
||||
"get_qwen_image_21_pre_process_func": get_qwen_image_21_pre_process_func,
|
||||
}[name]
|
||||
raise AttributeError(name)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,190 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CFG Parallel Mixin for Qwen-Image 2.1.
|
||||
|
||||
Differences from the 2.0 mixin (``qwen_image/cfg_parallel.py``):
|
||||
- The transformer owns a KV cache for the text/condition-image prefix, so the
|
||||
denoise loop passes a fresh per-generation ``kv_cache`` and names the CFG
|
||||
branch (``cache_branch="cond"/"uncond"``) on every forward.
|
||||
- The joint sequence is ``[condition images, ..., target image]`` — the target
|
||||
noise prediction is the *tail* of the output, not the head.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import (
|
||||
CFGParallelMixin,
|
||||
_get_cfg_world_size_or_one,
|
||||
_unwrap,
|
||||
_wrap,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_cfg_group,
|
||||
get_classifier_free_guidance_rank,
|
||||
)
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class QwenImage21CFGParallelMixin(CFGParallelMixin, ProgressBarMixin):
|
||||
"""Base Mixin class for the Qwen-Image 2.1 pipeline providing shared CFG methods."""
|
||||
|
||||
def predict_noise_maybe_with_cfg(
|
||||
self,
|
||||
do_true_cfg: bool,
|
||||
true_cfg_scale: float,
|
||||
positive_kwargs: dict[str, Any],
|
||||
negative_kwargs: dict[str, Any] | None,
|
||||
cfg_normalize: bool = False,
|
||||
output_slice: int | None = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
# The base implementation slices `[:, :output_slice]` per branch, which
|
||||
# matches the 2.0 condition-last layout. 2.1 concatenates condition
|
||||
# latents *before* the target, so the target prediction is the tail of
|
||||
# the joint output. Each branch must be tail-sliced *before* the CFG
|
||||
# combine: positive and negative prompts tokenize to different lengths,
|
||||
# so the full joint outputs do not align positionally.
|
||||
def tail_slice(pred: torch.Tensor | tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, ...]:
|
||||
if output_slice is None:
|
||||
return _wrap(pred)
|
||||
return tuple(p[:, -output_slice:] for p in _wrap(pred))
|
||||
|
||||
if do_true_cfg:
|
||||
if _get_cfg_world_size_or_one() > 1:
|
||||
cfg_group = get_cfg_group()
|
||||
cfg_rank = get_classifier_free_guidance_rank()
|
||||
branch_kwargs = positive_kwargs if cfg_rank == 0 else negative_kwargs
|
||||
local_pred = tail_slice(self.predict_noise(**branch_kwargs))
|
||||
gathered = [cfg_group.all_gather(p, separate_tensors=True) for p in local_pred]
|
||||
positive_noise_pred = tuple(g[0] for g in gathered)
|
||||
negative_noise_pred = tuple(g[1] for g in gathered)
|
||||
else:
|
||||
positive_noise_pred = tail_slice(self.predict_noise(**positive_kwargs))
|
||||
negative_noise_pred = tail_slice(self.predict_noise(**negative_kwargs))
|
||||
return self.combine_cfg_noise(
|
||||
positive_noise_pred,
|
||||
negative_noise_pred,
|
||||
true_cfg_scale,
|
||||
cfg_normalize,
|
||||
**({} if kwargs is None else {"kwargs": kwargs}),
|
||||
)
|
||||
|
||||
pred = self.predict_noise(**positive_kwargs)
|
||||
if output_slice is not None:
|
||||
pred = _unwrap(tail_slice(pred))
|
||||
return pred
|
||||
|
||||
def diffuse(
|
||||
self,
|
||||
prompt_embeds: torch.Tensor,
|
||||
prompt_embeds_mask: torch.Tensor,
|
||||
negative_prompt_embeds: torch.Tensor | None,
|
||||
negative_prompt_embeds_mask: torch.Tensor | None,
|
||||
img_mask: torch.Tensor,
|
||||
negative_img_mask: torch.Tensor | None,
|
||||
latents: torch.Tensor,
|
||||
image_latents: torch.Tensor | None,
|
||||
img_shapes: list,
|
||||
timesteps: torch.Tensor,
|
||||
do_true_cfg: bool,
|
||||
true_cfg_scale: float,
|
||||
attention_kwargs: dict[str, Any] | None = None,
|
||||
additional_transformer_kwargs: dict[str, Any] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Diffusion loop with optional classifier-free guidance.
|
||||
|
||||
Args:
|
||||
prompt_embeds: Positive prompt embeddings
|
||||
prompt_embeds_mask: Mask for positive prompt
|
||||
negative_prompt_embeds: Negative prompt embeddings
|
||||
negative_prompt_embeds_mask: Mask for negative prompt
|
||||
img_mask: Joint-sequence image-slot mask for the positive branch
|
||||
(VLM image slots plus one slot per 2x2 group of target tokens)
|
||||
negative_img_mask: Same for the negative branch
|
||||
latents: Noise latents to denoise (target image only, packed)
|
||||
image_latents: Packed condition-image latents, prepended to the
|
||||
noise latents (default: None for pure text-to-image)
|
||||
img_shapes: Per-sample list of (frame, height, width) latent-token
|
||||
shapes, condition images first and the target last
|
||||
timesteps: Diffusion timesteps
|
||||
do_true_cfg: Whether to apply CFG
|
||||
true_cfg_scale: CFG scale factor
|
||||
attention_kwargs: Passed through to the transformer (attn_path hint)
|
||||
additional_transformer_kwargs: Extra kwargs for the transformer
|
||||
|
||||
Returns:
|
||||
Denoised latents
|
||||
"""
|
||||
# Fresh cache for every generation: the pipeline is a singleton shared
|
||||
# across requests, so cache state must never leak between them.
|
||||
cache_enabled = getattr(self.transformer, "causal_condition", False)
|
||||
kv_cache = [{} for _ in range(len(self.transformer.transformer_blocks))] if cache_enabled else None
|
||||
self.scheduler.set_begin_index(0)
|
||||
self.transformer.do_true_cfg = do_true_cfg
|
||||
additional_transformer_kwargs = additional_transformer_kwargs or {}
|
||||
|
||||
with self.progress_bar(total=len(timesteps)) as pbar:
|
||||
for i, t in enumerate(timesteps):
|
||||
if self.interrupt:
|
||||
continue
|
||||
self._current_timestep = t
|
||||
|
||||
# Broadcast timestep to match batch size
|
||||
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
|
||||
|
||||
# Condition image latents go in front of the noise latents.
|
||||
latent_model_input = latents
|
||||
if image_latents is not None:
|
||||
latent_model_input = torch.cat([image_latents, latents], dim=1)
|
||||
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep / 1000,
|
||||
"encoder_hidden_states_mask": prompt_embeds_mask,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"img_shapes": img_shapes,
|
||||
"img_mask": img_mask,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"kv_cache": kv_cache,
|
||||
"cache_branch": "cond",
|
||||
**additional_transformer_kwargs,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep / 1000,
|
||||
"encoder_hidden_states_mask": negative_prompt_embeds_mask,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"img_shapes": img_shapes,
|
||||
"img_mask": negative_img_mask,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"kv_cache": kv_cache,
|
||||
"cache_branch": "uncond",
|
||||
**additional_transformer_kwargs,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
# Prefill (first step) returns the full joint sequence and
|
||||
# caches the prefix; decode steps return target tokens only.
|
||||
# Slicing the target tail is correct for both.
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg,
|
||||
true_cfg_scale,
|
||||
positive_kwargs,
|
||||
negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
output_slice=latents.size(1),
|
||||
)
|
||||
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
|
||||
pbar.update()
|
||||
|
||||
return latents
|
||||
@@ -0,0 +1,457 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CUDA Graph capture for Qwen-Image-2.1 KV-cache decode steps.
|
||||
|
||||
Decode (denoising steps 1..N) recomputes only the target image's tokens: the
|
||||
shape is identical every step and the timestep-independent prefix K/V come from
|
||||
the cross-step KV cache. That is the textbook CUDA-graph case, except the
|
||||
default cache stores prefix K/V in freshly allocated tensors, so a captured
|
||||
graph would bind addresses that the next request (or a step-mode cache merge)
|
||||
no longer owns.
|
||||
|
||||
Requests retain ownership of their prefix K/V. Decode refreshes separate
|
||||
fixed-address graph buffers when the active prefix tensors change. Graphs
|
||||
are keyed by branch, batch, prefix length, complete image layout,
|
||||
dtype, device and backend, so equal token counts with different RoPE layouts stay separate.
|
||||
|
||||
Each key admits a single live owner, tracked by a weak reference to the
|
||||
registering request's prefix K tensor. A second in-flight request whose cache
|
||||
aliases the same key is refused graph decode and stays eager until the
|
||||
owner's cache is released, so shared static buffers are never overwritten
|
||||
under a concurrent owner.
|
||||
|
||||
Memory note: graph entries own an additional copy of the prefix K/V. Decode
|
||||
attention concatenates that prefix with target K/V inside the captured region;
|
||||
transient tensors use the graph memory pool.
|
||||
|
||||
The model runner regionally torch.compile's the transformer blocks even when
|
||||
decode graphs are enabled, so capture records the compiled (fused) kernels;
|
||||
inductor's own cudagraphs stay off so the two graph layers never stack.
|
||||
|
||||
Fallbacks (all logged, all eager): CUDA unavailable, model-level CPU offload
|
||||
without persistent weight staging, sequence/ring/tensor parallelism,
|
||||
HSDP/offload hooks, KV-cache quantization, dynamic LoRA wrappers, padded text
|
||||
masks (the masked attention path branches on mask contents, which cannot be
|
||||
captured), unknown graph key at decode, a second live request aliasing an
|
||||
owned graph key, and capture failure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import weakref
|
||||
from collections import OrderedDict
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.models.qwen_image_21.qwen_image_21_transformer import (
|
||||
QwenImage21Transformer2DModel,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _dynamic_lora_wrappers_present(module: torch.nn.Module) -> bool:
|
||||
"""True once ``DiffusionLoRAManager`` has wrapped layers under ``module``.
|
||||
|
||||
A captured graph records the layers and branches that ran at capture time;
|
||||
binding or rescaling an adapter afterwards changes the math without
|
||||
changing the shapes the graph was keyed on, so graphs and dynamic LoRA
|
||||
must not mix. Same reasoning as ``sensenova_u1.paged_decode``.
|
||||
"""
|
||||
try:
|
||||
from vllm.lora.layers import BaseLayerWithLoRA
|
||||
except ImportError: # pragma: no cover - depends on the wheel
|
||||
return False
|
||||
return any(isinstance(m, BaseLayerWithLoRA) for m in module.modules())
|
||||
|
||||
|
||||
class QwenImage21DecodeGraphEntry:
|
||||
"""Static state for one captured decode graph.
|
||||
|
||||
Buffers are created outside inference mode so replay-side ``copy_`` is
|
||||
legal regardless of the caller's grad context. ``graph`` is None until the
|
||||
first decode step captures it. Failed entries are replaced by a ``None``
|
||||
marker in the manager so their buffers can be released.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
branch: str,
|
||||
batch_size: int,
|
||||
prefix_len: int,
|
||||
target_tokens: int,
|
||||
in_channels: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
kv_shapes: list[torch.Size],
|
||||
kv_dtype: torch.dtype,
|
||||
freqs: torch.Tensor,
|
||||
):
|
||||
self.branch = branch
|
||||
self.target_tokens = target_tokens
|
||||
self.prefix_len = prefix_len
|
||||
with torch.inference_mode(False):
|
||||
self.hidden = torch.zeros(batch_size, target_tokens, in_channels, dtype=dtype, device=device)
|
||||
self.timestep = torch.zeros(batch_size, dtype=dtype, device=device)
|
||||
self.k = [torch.zeros(shape, dtype=kv_dtype, device=device) for shape in kv_shapes]
|
||||
self.v = [torch.zeros(shape, dtype=kv_dtype, device=device) for shape in kv_shapes]
|
||||
self.target_token_mask = torch.ones(target_tokens, dtype=torch.bool, device=device)
|
||||
# RoPE frequencies are layout-dependent but step-independent. Clone
|
||||
# into a normal (non-inference) tensor: the prefill runs under
|
||||
# inference_mode, and the captured body takes views of this tensor
|
||||
# outside it, which is forbidden for inference tensors.
|
||||
self.freqs = freqs.clone()
|
||||
self.attn_metadata = None
|
||||
# Per-block cache dicts with the same shape as the legacy protocol.
|
||||
self.block_caches = [{branch: {"key": self.k[i], "value": self.v[i]}} for i in range(len(kv_shapes))]
|
||||
self.graph: torch.cuda.CUDAGraph | None = None
|
||||
self.output: torch.Tensor | None = None
|
||||
self.captures = 0
|
||||
self.prefix_sources: list[tuple[weakref.ReferenceType[torch.Tensor], int | None]] = []
|
||||
|
||||
def refresh_prefix(self, kv_cache: list[dict[str, dict[str, torch.Tensor]]]) -> None:
|
||||
sources = [block[self.branch][name] for block in kv_cache for name in ("key", "value")]
|
||||
versions = [None if tensor.is_inference() else tensor._version for tensor in sources]
|
||||
if len(sources) == len(self.prefix_sources) and all(
|
||||
previous() is tensor and version == current_version
|
||||
for (previous, version), tensor, current_version in zip(self.prefix_sources, sources, versions)
|
||||
):
|
||||
return
|
||||
# Prefill replaces inference-mode tensors; decode treats them as immutable.
|
||||
# Weak references avoid retaining completed requests or mistaking a reused
|
||||
# allocation for the same prefix. Normal tensors also track in-place writes.
|
||||
self.prefix_sources = []
|
||||
for i, block in enumerate(kv_cache):
|
||||
self.k[i].copy_(block[self.branch]["key"])
|
||||
self.v[i].copy_(block[self.branch]["value"])
|
||||
self.prefix_sources = [(weakref.ref(tensor), version) for tensor, version in zip(sources, versions)]
|
||||
|
||||
def capture(self, body) -> None:
|
||||
"""Warm up on a side stream, then capture one decode step.
|
||||
|
||||
Warm-up runs the same computation it will record, allocating cuBLAS
|
||||
workspaces and any lazy buffers outside the capture. Decode writes
|
||||
nothing to the cache (prefix K/V are read-only), so warm-up is
|
||||
idempotent.
|
||||
"""
|
||||
side = torch.cuda.Stream()
|
||||
side.wait_stream(torch.cuda.current_stream())
|
||||
try:
|
||||
with torch.cuda.stream(side), torch.inference_mode(False), torch.no_grad():
|
||||
for _ in range(2):
|
||||
body()
|
||||
finally:
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
# The platform-wide pool is shared with every other captured path in
|
||||
# the tree; entries only keep their *output* alive, whose tiny size
|
||||
# makes reuse safe because the caller clones it before any other graph
|
||||
# on the pool replays.
|
||||
with torch.inference_mode(False), torch.no_grad():
|
||||
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
|
||||
self.output = body()
|
||||
self.graph = graph
|
||||
self.captures += 1
|
||||
|
||||
|
||||
class QwenImage21DecodeGraphManager:
|
||||
"""Owns the static KV buffers and captured graphs of one transformer."""
|
||||
|
||||
def __init__(self, model: QwenImage21Transformer2DModel, max_entries: int = 8, model_level_offload: bool = False):
|
||||
self.model = model
|
||||
self.max_entries = max_entries
|
||||
self.model_level_offload = model_level_offload
|
||||
self.entries: OrderedDict[tuple, QwenImage21DecodeGraphEntry | None] = OrderedDict()
|
||||
# Key -> weak reference to the prefix K tensor of the request that owns
|
||||
# the entry's static buffers. Dead references mean the owning request
|
||||
# finished and its cache was released, so the key is free to adopt.
|
||||
self._owners: dict[tuple, weakref.ReferenceType[torch.Tensor]] = {}
|
||||
self._static_eligible: bool | None = None
|
||||
|
||||
# ── eligibility ──
|
||||
|
||||
def _offload_reason(self) -> str | None:
|
||||
"""Reason decode graphs are incompatible with model-level offload, if any.
|
||||
|
||||
Model-level (sequential) offload registers its hook on the top-level
|
||||
transformer module, not on individual blocks. With persistent staging
|
||||
the hook keeps the DiT on fixed device storage and only rebinds
|
||||
``p.data`` around each generation, and the swap completes in
|
||||
``pre_forward`` before capture or replay begin, so captured weight
|
||||
pointers stay valid. Without staging every swap allocates fresh
|
||||
storage and replay would read stale pointers, so capture stays
|
||||
disabled and decode falls back to eager.
|
||||
"""
|
||||
registry = getattr(self.model, "_hook_registry", None)
|
||||
hooks = registry._hooks if registry is not None else {}
|
||||
if hooks:
|
||||
from vllm_omni.diffusion.offloader.sequential_backend import (
|
||||
SequentialOffloadHook,
|
||||
sequential_offload_staging_active,
|
||||
)
|
||||
|
||||
if set(hooks) <= {SequentialOffloadHook._HOOK_NAME} and sequential_offload_staging_active(self.model):
|
||||
return None
|
||||
return "model module carries offload/cache hooks"
|
||||
if self.model_level_offload:
|
||||
return "model-level CPU offload without persistent staging swaps weight storage after capture"
|
||||
return None
|
||||
|
||||
def _check_static_eligibility(self) -> bool:
|
||||
"""One-time checks that cannot change after weights are loaded."""
|
||||
model = self.model
|
||||
reason = self._offload_reason()
|
||||
param = next(model.parameters(), None)
|
||||
if reason is None and (param is None or param.device.type != "cuda"):
|
||||
reason = f"model is not on CUDA (device={None if param is None else param.device})"
|
||||
elif reason is None:
|
||||
parallel_config = getattr(model, "parallel_config", None)
|
||||
if parallel_config is not None:
|
||||
sp = getattr(parallel_config, "sequence_parallel_size", None) or 1
|
||||
if sp > 1 or getattr(parallel_config, "ring_degree", 1) > 1:
|
||||
reason = "sequence/ring parallelism changes decode shapes per rank"
|
||||
elif getattr(parallel_config, "use_hsdp", False):
|
||||
reason = "HSDP wraps blocks with gather hooks outside the captured region"
|
||||
if reason is None:
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
|
||||
if get_tensor_model_parallel_world_size() > 1:
|
||||
reason = "tensor parallelism runs collectives inside the captured region"
|
||||
if reason is None:
|
||||
for block in model.transformer_blocks:
|
||||
if hasattr(block, "_hook_registry") or hasattr(block, "_omni_original_forward"):
|
||||
reason = "transformer blocks carry offload/cache hooks"
|
||||
break
|
||||
if reason is None and model.transformer_blocks:
|
||||
attn_layer = model.transformer_blocks[0].attn.attn
|
||||
if getattr(attn_layer, "_kv_cache_dtype", None) is not None:
|
||||
reason = "KV-cache quantization is enabled"
|
||||
|
||||
if reason is not None:
|
||||
logger.warning_once("Qwen-Image-2.1 CUDA graph decode disabled: %s. Falling back to eager decode.", reason)
|
||||
return False
|
||||
return True
|
||||
|
||||
def eligible(self) -> bool:
|
||||
if self._static_eligible is None:
|
||||
self._static_eligible = self._check_static_eligibility()
|
||||
return self._static_eligible
|
||||
|
||||
def _backend_name(self) -> str:
|
||||
backend = self.model.transformer_blocks[0].attn.attn.attn_backend
|
||||
return backend.get_name() if backend is not None else "custom"
|
||||
|
||||
# ── prefill registration ──
|
||||
|
||||
def register_prefill(
|
||||
self,
|
||||
*,
|
||||
kv_cache: list[dict[str, dict[str, torch.Tensor]]],
|
||||
cache_branch: str,
|
||||
prefix_len: int,
|
||||
img_shapes: list[tuple[int, int, int]],
|
||||
target_freqs: torch.Tensor,
|
||||
joint_key_valid: torch.Tensor | None,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Prepare graph buffers without changing the request's prefix K/V.
|
||||
|
||||
The first in-flight request to register a key owns its static buffers;
|
||||
registrations for the same key from another live request are refused
|
||||
so their decode stays eager until the owner completes.
|
||||
"""
|
||||
if not self.eligible() or torch.compiler.is_compiling():
|
||||
return
|
||||
if _dynamic_lora_wrappers_present(self.model):
|
||||
if self.entries:
|
||||
logger.warning_once(
|
||||
"Qwen-Image-2.1 CUDA graph decode: LoRA wrappers appeared; "
|
||||
"dropping captured graphs and falling back to eager decode."
|
||||
)
|
||||
self.entries.clear()
|
||||
return
|
||||
if joint_key_valid is not None and not bool(joint_key_valid.all()):
|
||||
# The masked decode path (varlen-unpack or 4D mask) branches on mask
|
||||
# contents and cannot be captured; this request stays eager.
|
||||
logger.warning_once(
|
||||
"Qwen-Image-2.1 CUDA graph decode: padded text mask present; this request falls back to eager decode."
|
||||
)
|
||||
return
|
||||
|
||||
first = kv_cache[0].get(cache_branch)
|
||||
if first is None or "key" not in first:
|
||||
return
|
||||
if any(name.endswith("_scale") for name in first):
|
||||
# Quantized (FP8) prefix storage: the static K/V buffers assume a single
|
||||
# dtype and the captured body would not see the dequant scales (a bf16 K +
|
||||
# fp8 V mix would even copy_ fp8 into bf16 buffers, silently dropping the
|
||||
# scales). Keep quantized-cache requests on the eager decode path.
|
||||
logger.warning_once(
|
||||
"Qwen-Image-2.1 CUDA graph decode: quantized prefix KV cache "
|
||||
"(prefix_kv_cache_dtype) is not graph-capturable; falling back to eager decode."
|
||||
)
|
||||
return
|
||||
batch_size = first["key"].shape[0]
|
||||
key = (
|
||||
cache_branch,
|
||||
batch_size,
|
||||
prefix_len,
|
||||
tuple(tuple(shape) for shape in img_shapes),
|
||||
dtype,
|
||||
first["key"].device.index,
|
||||
self._backend_name(),
|
||||
)
|
||||
|
||||
if key not in self.entries:
|
||||
while len(self.entries) >= self.max_entries:
|
||||
evicted_key, evicted_entry = self.entries.popitem(last=False)
|
||||
self._owners.pop(evicted_key, None)
|
||||
del evicted_entry
|
||||
logger.debug("Evicting decode graph entry %s (max_entries=%d)", evicted_key, self.max_entries)
|
||||
allocation_error = None
|
||||
try:
|
||||
self.entries[key] = QwenImage21DecodeGraphEntry(
|
||||
branch=cache_branch,
|
||||
batch_size=batch_size,
|
||||
prefix_len=prefix_len,
|
||||
target_tokens=target_freqs.shape[0],
|
||||
in_channels=self.model.in_channels,
|
||||
dtype=dtype,
|
||||
device=first["key"].device,
|
||||
kv_shapes=[block_cache[cache_branch]["key"].shape for block_cache in kv_cache],
|
||||
kv_dtype=first["key"].dtype,
|
||||
freqs=target_freqs,
|
||||
)
|
||||
except torch.OutOfMemoryError as exc:
|
||||
allocation_error = str(exc)
|
||||
if allocation_error is not None:
|
||||
# Keep a bounded failure marker without retaining tensors or the exception traceback.
|
||||
self.entries[key] = None
|
||||
current_omni_platform.empty_cache()
|
||||
logger.warning(
|
||||
"Qwen-Image-2.1 CUDA graph allocation failed for key=%s: %s. Falling back to eager decode.",
|
||||
key,
|
||||
allocation_error,
|
||||
)
|
||||
return
|
||||
else:
|
||||
self.entries.move_to_end(key)
|
||||
|
||||
entry = self.entries[key]
|
||||
if entry is not None:
|
||||
owner = self._owners.get(key)
|
||||
owner_prefix = owner() if owner is not None else None
|
||||
if owner_prefix is not None and owner_prefix is not first["key"]:
|
||||
# Another in-flight request still owns this entry's static
|
||||
# buffers; refreshing them would overwrite that request's
|
||||
# prefix mid-generation. This request decodes eagerly until
|
||||
# the owner's cache is released and the weak reference dies.
|
||||
logger.warning_once(
|
||||
"Qwen-Image-2.1 CUDA graph decode: entry key=%s is owned by another in-flight "
|
||||
"request; this request falls back to eager decode.",
|
||||
key,
|
||||
)
|
||||
return
|
||||
self._owners[key] = weakref.ref(first["key"])
|
||||
entry.prefix_sources = []
|
||||
|
||||
logger.debug(
|
||||
"Registered decode graph entry key=%s (entries=%d, captured=%d)",
|
||||
key,
|
||||
len(self.entries),
|
||||
sum(1 for e in self.entries.values() if e is not None and e.graph is not None),
|
||||
)
|
||||
|
||||
# ── decode replay ──
|
||||
|
||||
def try_decode(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
kv_cache: list[dict[str, dict[str, torch.Tensor]]],
|
||||
cache_branch: str,
|
||||
img_shapes: list[tuple[int, int, int]],
|
||||
img_mask: torch.Tensor,
|
||||
encoder_hidden_states_mask: torch.Tensor | None,
|
||||
) -> torch.Tensor | None:
|
||||
"""Replay the captured decode step, or return None to fall back to eager."""
|
||||
if not self.eligible() or torch.compiler.is_compiling():
|
||||
return None
|
||||
if _dynamic_lora_wrappers_present(self.model):
|
||||
self.clear()
|
||||
return None
|
||||
if encoder_hidden_states_mask is not None:
|
||||
text_positions = ~img_mask[0, : encoder_hidden_states_mask.shape[1]]
|
||||
if not bool(encoder_hidden_states_mask[:, text_positions].all()):
|
||||
return None
|
||||
first = kv_cache[0][cache_branch]
|
||||
cached_key = first["key"]
|
||||
if cached_key.device.type != "cuda":
|
||||
# Registration is device-agnostic (unit tests exercise the buffer
|
||||
# machinery on CPU), but capture/replay is CUDA-only: an empty
|
||||
# graph over CPU ops would replay as a stale no-op.
|
||||
return None
|
||||
key = (
|
||||
cache_branch,
|
||||
cached_key.shape[0],
|
||||
cached_key.shape[1],
|
||||
tuple(tuple(shape) for shape in img_shapes),
|
||||
hidden_states.dtype,
|
||||
cached_key.device.index,
|
||||
self._backend_name(),
|
||||
)
|
||||
entry = self.entries.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
self.entries.move_to_end(key)
|
||||
|
||||
owner = self._owners.get(key)
|
||||
owner_prefix = owner() if owner is not None else None
|
||||
if owner_prefix is not None and owner_prefix is not cached_key:
|
||||
# The entry is owned by a different in-flight request (see
|
||||
# register_prefill); this request stays on the eager decode path
|
||||
# until the owner's cache is released.
|
||||
return None
|
||||
|
||||
entry.hidden.copy_(hidden_states[:, -entry.target_tokens :])
|
||||
entry.timestep.copy_(timestep)
|
||||
# Graph buffers are scratch space; request caches must never alias them.
|
||||
entry.refresh_prefix(kv_cache)
|
||||
|
||||
if entry.graph is None:
|
||||
capture_error = None
|
||||
try:
|
||||
entry.capture(lambda: self.model._decode_graph_forward(entry))
|
||||
except Exception as exc:
|
||||
capture_error = str(exc)
|
||||
if capture_error is not None:
|
||||
self.entries[key] = None
|
||||
del entry
|
||||
current_omni_platform.empty_cache()
|
||||
logger.warning(
|
||||
"Qwen-Image-2.1 CUDA graph capture failed for key=%s: %s. "
|
||||
"Falling back to eager decode for this shape.",
|
||||
key,
|
||||
capture_error,
|
||||
)
|
||||
return None
|
||||
logger.info("Captured Qwen-Image-2.1 decode graph for key=%s", key)
|
||||
entry.graph.replay()
|
||||
assert entry.output is not None
|
||||
# Clone before returning: the output lives in the shared graph pool and
|
||||
# the next replay (of this or another graph on the pool) overwrites it.
|
||||
return entry.output.clone()
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Drop all entries, freeing buffers and captured graphs."""
|
||||
self.entries.clear()
|
||||
self._owners.clear()
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -98,6 +98,12 @@ class PinnedModuleStager:
|
||||
A module iterable is treated as one staging group. It uses one copy stream
|
||||
and one reusable completion event. Tensors sharing storage keep their
|
||||
shapes, strides, offsets, dtypes, and aliases across every transition.
|
||||
|
||||
With ``retain_device_storage`` the device storages survive ``offload``:
|
||||
``load`` then only rebinds to the same fixed storages (weights are
|
||||
immutable, so no H2D copy repeats). The stable storage addresses keep
|
||||
CUDA-graph-captured weight pointers valid across offload/load swaps, at
|
||||
the cost of keeping the device storage resident while offloaded.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -108,6 +114,7 @@ class PinnedModuleStager:
|
||||
pin_memory: bool = True,
|
||||
copy_stream: Any | None = None,
|
||||
cache_retention: BoundedAllocatorCache | None = None,
|
||||
retain_device_storage: bool = False,
|
||||
) -> None:
|
||||
modules = (module,) if isinstance(module, nn.Module) else tuple(module)
|
||||
if not modules or not all(isinstance(item, nn.Module) for item in modules):
|
||||
@@ -117,6 +124,7 @@ class PinnedModuleStager:
|
||||
self.copy_stream = copy_stream if copy_stream is not None else current_omni_platform.Stream()
|
||||
self._ready_event = current_omni_platform.Event()
|
||||
self.cache_retention = cache_retention
|
||||
self.retain_device_storage = retain_device_storage
|
||||
self.loaded = False
|
||||
self._groups = self._snapshot_groups(modules, pin_memory=pin_memory)
|
||||
self._device_storages: list[torch.Tensor] = []
|
||||
@@ -234,6 +242,12 @@ class PinnedModuleStager:
|
||||
def load(self) -> None:
|
||||
if self.loaded:
|
||||
return
|
||||
if self._device_storages:
|
||||
# Storage retained from a previous load still holds the immutable
|
||||
# weights; rebinding suffices, no H2D copy repeats.
|
||||
self._bind(self._device_storages)
|
||||
self.loaded = True
|
||||
return
|
||||
try:
|
||||
self._load_once()
|
||||
except torch.OutOfMemoryError:
|
||||
@@ -263,7 +277,8 @@ class PinnedModuleStager:
|
||||
self.loaded = False
|
||||
self._release_cache(force=True)
|
||||
raise
|
||||
self._device_storages.clear()
|
||||
if not self.retain_device_storage:
|
||||
self._device_storages.clear()
|
||||
self.loaded = False
|
||||
self._release_cache()
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm_omni.diffusion.hooks import HookRegistry, ModelHook
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .base import OffloadBackend, OffloadConfig, SupportsModelCpuOffload
|
||||
from .module_residency import PinnedModuleStager
|
||||
from .plan_resolver import resolve_offload_plan
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -60,6 +61,7 @@ class SequentialOffloadHook(ModelHook):
|
||||
pin_memory: bool = True,
|
||||
use_hsdp: bool = False,
|
||||
offload_after_context: bool = True,
|
||||
persistent_staging: bool = False,
|
||||
):
|
||||
# Modules to offload to CPU before this module runs
|
||||
self.offload_targets = offload_targets
|
||||
@@ -67,6 +69,21 @@ class SequentialOffloadHook(ModelHook):
|
||||
self.pin_memory = pin_memory
|
||||
self.use_hsdp = use_hsdp
|
||||
self.offload_after_context = offload_after_context
|
||||
self.persistent_staging = persistent_staging
|
||||
self._stager: PinnedModuleStager | None = None
|
||||
|
||||
def initialize_hook(self, module: nn.Module) -> nn.Module:
|
||||
if self.persistent_staging and not self.use_hsdp:
|
||||
# Fixed staging storage keeps CUDA-graph-captured weight pointers
|
||||
# valid across offload/load swaps. HSDP keeps the plain move
|
||||
# semantics (decode graphs refuse HSDP anyway).
|
||||
self._stager = PinnedModuleStager(
|
||||
module,
|
||||
self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
retain_device_storage=True,
|
||||
)
|
||||
return module
|
||||
|
||||
@staticmethod
|
||||
def _move_params(
|
||||
@@ -101,7 +118,23 @@ class SequentialOffloadHook(ModelHook):
|
||||
moved = True
|
||||
return moved
|
||||
|
||||
@staticmethod
|
||||
def _module_stager(module: nn.Module) -> PinnedModuleStager | None:
|
||||
"""The persistent stager owned by ``module``'s own sequential hook, if any.
|
||||
|
||||
Cross-module swaps call ``_to_cpu(target)`` through the *running*
|
||||
module's hook, so staging state must be resolved on the target's hook
|
||||
to keep its ``loaded`` bookkeeping consistent.
|
||||
"""
|
||||
registry: HookRegistry | None = getattr(module, "_hook_registry", None)
|
||||
hook = registry.get_hook(SequentialOffloadHook._HOOK_NAME) if registry is not None else None
|
||||
return hook._stager if isinstance(hook, SequentialOffloadHook) else None
|
||||
|
||||
def _to_cpu(self, module: nn.Module) -> None:
|
||||
stager = self._module_stager(module)
|
||||
if stager is not None:
|
||||
stager.offload()
|
||||
return
|
||||
# XPU's allocator doesn't respect stream dependencies in empty_cache,
|
||||
# so non-blocking copies can race with cache eviction. Use blocking
|
||||
# copies on XPU to avoid NULL pointer errors during DMA.
|
||||
@@ -116,6 +149,10 @@ class SequentialOffloadHook(ModelHook):
|
||||
current_omni_platform.empty_cache()
|
||||
|
||||
def _to_gpu(self, module: nn.Module) -> None:
|
||||
stager = self._module_stager(module)
|
||||
if stager is not None:
|
||||
stager.load()
|
||||
return
|
||||
self._move_params(module, self.device, non_blocking=False)
|
||||
|
||||
def pre_forward(self, module: nn.Module, *args, **kwargs) -> tuple[tuple, dict]:
|
||||
@@ -147,6 +184,7 @@ def apply_sequential_offload(
|
||||
offload_initial_dits: bool = False,
|
||||
offload_dit_modules: Collection[nn.Module] | None = None,
|
||||
offload_encoder_modules: Collection[nn.Module] | None = None,
|
||||
persistent_dit_staging: bool = False,
|
||||
) -> None:
|
||||
"""Apply sequential offloading hooks to DiT and encoder modules.
|
||||
|
||||
@@ -165,6 +203,10 @@ def apply_sequential_offload(
|
||||
every DiT for backward compatibility.
|
||||
offload_encoder_modules: Encoder/stage modules allowed to move to CPU.
|
||||
None selects every supplied module for backward compatibility.
|
||||
persistent_dit_staging: Keep DiT weights on fixed device staging
|
||||
storage across swaps, so CUDA-graph-captured weight pointers stay
|
||||
valid. Encoders keep plain move semantics so their memory is
|
||||
actually freed while the DiT runs.
|
||||
|
||||
Example:
|
||||
>>> apply_sequential_offload(
|
||||
@@ -193,6 +235,7 @@ def apply_sequential_offload(
|
||||
pin_memory=pin_memory,
|
||||
use_hsdp=use_hsdp,
|
||||
offload_after_context=id(dit_mod) in selected_dit_ids,
|
||||
persistent_staging=persistent_dit_staging,
|
||||
)
|
||||
registry.register_hook(SequentialOffloadHook._HOOK_NAME, hook)
|
||||
logger.debug("Registered offload hook for %s", dit_mod.__class__.__name__)
|
||||
@@ -261,6 +304,15 @@ def _get_sequential_offload_hook(module: nn.Module) -> SequentialOffloadHook:
|
||||
return hook
|
||||
|
||||
|
||||
def sequential_offload_staging_active(module: nn.Module) -> bool:
|
||||
"""True when the module's sequential offload hook serves fixed device storage.
|
||||
|
||||
Fixed staging storage keeps CUDA-graph-captured weight pointers valid
|
||||
across offload/load swaps.
|
||||
"""
|
||||
return SequentialOffloadHook._module_stager(module) is not None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def sequential_offload_component(module: nn.Module) -> Iterator[None]:
|
||||
"""Activate and release a hooked component called outside ``forward``."""
|
||||
@@ -362,6 +414,14 @@ class ModelLevelOffloadBackend(OffloadBackend):
|
||||
use_hsdp=self.config.use_hsdp,
|
||||
offload_dit_modules=selected_dits,
|
||||
offload_encoder_modules=selected_encoders,
|
||||
# Fixed DiT staging keeps decode-graph weight pointers valid;
|
||||
# only models that capture weight pointers (Qwen-Image-2.1's
|
||||
# CUDA-graph decode) pay for it — every other model keeps
|
||||
# plain move semantics so offloading actually frees DiT VRAM
|
||||
# while the encoders run.
|
||||
persistent_dit_staging=(
|
||||
self.device.type == "cuda" and any(getattr(dit, "enable_cuda_graph_decode", False) for dit in dits)
|
||||
),
|
||||
)
|
||||
except BaseException:
|
||||
try:
|
||||
|
||||
@@ -56,6 +56,11 @@ _DIFFUSION_MODELS = {
|
||||
"pipeline_qwen_image_layered",
|
||||
"QwenImageLayeredPipeline",
|
||||
),
|
||||
"QwenImage21Pipeline": (
|
||||
"qwen_image_21",
|
||||
"pipeline_qwen_image_21",
|
||||
"QwenImage21Pipeline",
|
||||
),
|
||||
"GlmImagePipeline": (
|
||||
"glm_image",
|
||||
"pipeline_glm_image",
|
||||
@@ -426,6 +431,10 @@ _NO_CACHE_ACCELERATION = {
|
||||
"Pi0Pipeline",
|
||||
"Pi05Pipeline",
|
||||
"LingBotWorldCausalDMDPipeline",
|
||||
# Qwen-Image 2.1's transformer carries its own prefix KV cache (keyed by CFG
|
||||
# branch) across denoising steps, which conflicts with cache_dit / tea_cache
|
||||
# step-skipping hooks.
|
||||
"QwenImage21Pipeline",
|
||||
}
|
||||
|
||||
|
||||
@@ -651,6 +660,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = {
|
||||
"AnimaPipeline": "get_anima_post_process_func",
|
||||
"QwenImageEditPipeline": "get_qwen_image_edit_post_process_func",
|
||||
"QwenImageEditPlusPipeline": "get_qwen_image_edit_plus_post_process_func",
|
||||
"QwenImage21Pipeline": "get_qwen_image_21_post_process_func",
|
||||
"GlmImagePipeline": "get_glm_image_post_process_func",
|
||||
"ZImagePipeline": "get_post_process_func",
|
||||
"OvisImagePipeline": "get_ovis_image_post_process_func",
|
||||
@@ -740,6 +750,7 @@ _DIFFUSION_PRE_PROCESS_FUNCS = {
|
||||
"BooguImageTurboPipeline": "get_boogu_image_pre_process_func",
|
||||
"QwenImageEditPipeline": "get_qwen_image_edit_pre_process_func",
|
||||
"QwenImageEditPlusPipeline": "get_qwen_image_edit_plus_pre_process_func",
|
||||
"QwenImage21Pipeline": "get_qwen_image_21_pre_process_func",
|
||||
"LongCatImageEditPipeline": "get_longcat_image_edit_pre_process_func",
|
||||
"LongCatVideoAvatarPipeline": "get_longcat_video_avatar_pre_process_func",
|
||||
"QwenImageLayeredPipeline": "get_qwen_image_layered_pre_process_func",
|
||||
|
||||
@@ -76,6 +76,8 @@ class OmniDiffusionRequest:
|
||||
# A model preprocessor may keep selected requests on the legacy full-forward
|
||||
# path even when the engine is globally configured for step execution.
|
||||
use_step_execution: bool = True
|
||||
# Whether first and later denoising steps may share a step batch.
|
||||
allow_mixed_step_phases: bool = True
|
||||
# KV-recv wall-clock (ms), set by the runner's _prepare_request_for_forward
|
||||
# and carried to DiffusionOutput for the vllm_omni:diffusion_kv_load_s metric.
|
||||
kv_recv_ms: float = 0.0
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm_omni.diffusion.sched.base_scheduler import BaseScheduler
|
||||
from vllm_omni.diffusion.sched.interface import (
|
||||
DiffusionRequestStatus,
|
||||
DiffusionSchedulerOutput,
|
||||
SchedulerRequestState,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -62,6 +63,19 @@ class StepScheduler(BaseScheduler):
|
||||
)
|
||||
return request_id
|
||||
|
||||
def _can_schedule_waiting(self, state: SchedulerRequestState) -> bool:
|
||||
if not super()._can_schedule_waiting(state):
|
||||
return False
|
||||
|
||||
is_first_step = self._request_progress[state.request_id].current_step == 0
|
||||
for request_id in self._running:
|
||||
running = self._request_states[request_id]
|
||||
if state.req.allow_mixed_step_phases and running.req.allow_mixed_step_phases:
|
||||
continue
|
||||
if (self._request_progress[request_id].current_step == 0) != is_first_step:
|
||||
return False
|
||||
return True
|
||||
|
||||
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: BaseRunnerOutput) -> set[str]:
|
||||
scheduled_request_ids = sched_output.scheduled_request_ids
|
||||
if not scheduled_request_ids and not sched_output.finished_req_ids:
|
||||
|
||||
@@ -206,7 +206,7 @@ class DiffusionModelRunner(DiffusionStagePayloadMixin):
|
||||
self.kv_transfer_manager = (
|
||||
payload_transfer_manager if getattr(od_config, "kv_transfer_config", None) is None else None
|
||||
)
|
||||
self.init_omni_connectors(od_config, payload_transfer_manager, synchronous=True)
|
||||
self.init_omni_connectors(od_config, payload_transfer_manager, synchronous=True) # type: ignore[arg-type]
|
||||
self._kv_connector = None
|
||||
from vllm_omni.diffusion.diffusion_kv.kv_connector import KVReceiveProgress, native_prefetch_enabled
|
||||
|
||||
@@ -250,14 +250,25 @@ class DiffusionModelRunner(DiffusionStagePayloadMixin):
|
||||
if model is None:
|
||||
return
|
||||
|
||||
compile_kwargs: dict[str, Any] = {"dynamic": self.od_config.diffusion_compile_dynamic}
|
||||
if getattr(model, "enable_cuda_graph_decode", False):
|
||||
# Decode-graph models still compile their blocks: graph capture
|
||||
# records the compiled (fused) kernels. Scope inductor cudagraphs
|
||||
# to this compile so the two graph layers never stack, without
|
||||
# changing torch.compile for every other model in the process.
|
||||
compile_kwargs["options"] = {
|
||||
"triton.cudagraphs": False,
|
||||
"triton.cudagraph_trees": False,
|
||||
}
|
||||
logger.info("Model runner: %s combines CUDA graph decode with torch.compile.", attr_name)
|
||||
|
||||
compile_granularity = self.od_config.diffusion_compile_granularity
|
||||
compile_dynamic = self.od_config.diffusion_compile_dynamic
|
||||
try:
|
||||
if compile_granularity == "full":
|
||||
model.compile(dynamic=compile_dynamic)
|
||||
model.compile(**compile_kwargs)
|
||||
compiled_model = model
|
||||
else:
|
||||
compiled_model = regionally_compile(model, dynamic=compile_dynamic)
|
||||
compiled_model = regionally_compile(model, **compile_kwargs)
|
||||
setattr(self.pipeline, attr_name, compiled_model)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
@@ -275,7 +286,7 @@ class DiffusionModelRunner(DiffusionStagePayloadMixin):
|
||||
"compilation errors may surface on the first request.",
|
||||
attr_name,
|
||||
compile_granularity,
|
||||
compile_dynamic,
|
||||
compile_kwargs["dynamic"],
|
||||
)
|
||||
|
||||
def load_model(
|
||||
@@ -1454,7 +1465,7 @@ class DiffusionModelRunner(DiffusionStagePayloadMixin):
|
||||
else req.denoise_completed
|
||||
)
|
||||
if finished and result is not None:
|
||||
self._maybe_send_stage_payload([req], [result])
|
||||
self._maybe_send_stage_payload([req], [result]) # type: ignore[list-item]
|
||||
runner_output_list.append(
|
||||
RunnerOutput(
|
||||
request_id=req.request_id,
|
||||
|
||||
Reference in new issue
Block a user