[New model] Qwen-Image-2.1 support (rebase + review follow-ups) (#8099)
Signed-off-by: NumberWan <wantszkin2003@gmail.com> Signed-off-by: SamitHuang <285365963@qq.com> Signed-off-by: Samit <285365963@qq.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: SamitHuang <285365963@qq.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
52 files changed
+8058
-136
No files matched your search
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Qwen Image 2.1 diffusion model components.
|
||||
|
||||
Lazy by necessity: the pipeline imports the distributed VAE
|
||||
(`vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_qwenimage21`),
|
||||
which in turn imports the plain VAE module from this package. Eager imports
|
||||
here would make that a circular import whenever the distributed module is
|
||||
imported first.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
__all__ = [
|
||||
"AutoencoderKLQwenImage21",
|
||||
"QwenImage21Pipeline",
|
||||
"QwenImage21Transformer2DModel",
|
||||
"get_qwen_image_21_post_process_func",
|
||||
"get_qwen_image_21_pre_process_func",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name == "AutoencoderKLQwenImage21":
|
||||
from .autoencoder_kl_qwenimage21 import AutoencoderKLQwenImage21
|
||||
|
||||
return AutoencoderKLQwenImage21
|
||||
if name == "QwenImage21Transformer2DModel":
|
||||
from .qwen_image_21_transformer import QwenImage21Transformer2DModel
|
||||
|
||||
return QwenImage21Transformer2DModel
|
||||
if name in (
|
||||
"QwenImage21Pipeline",
|
||||
"get_qwen_image_21_post_process_func",
|
||||
"get_qwen_image_21_pre_process_func",
|
||||
):
|
||||
from .pipeline_qwen_image_21 import (
|
||||
QwenImage21Pipeline,
|
||||
get_qwen_image_21_post_process_func,
|
||||
get_qwen_image_21_pre_process_func,
|
||||
)
|
||||
|
||||
return {
|
||||
"QwenImage21Pipeline": QwenImage21Pipeline,
|
||||
"get_qwen_image_21_post_process_func": get_qwen_image_21_post_process_func,
|
||||
"get_qwen_image_21_pre_process_func": get_qwen_image_21_pre_process_func,
|
||||
}[name]
|
||||
raise AttributeError(name)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,190 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CFG Parallel Mixin for Qwen-Image 2.1.
|
||||
|
||||
Differences from the 2.0 mixin (``qwen_image/cfg_parallel.py``):
|
||||
- The transformer owns a KV cache for the text/condition-image prefix, so the
|
||||
denoise loop passes a fresh per-generation ``kv_cache`` and names the CFG
|
||||
branch (``cache_branch="cond"/"uncond"``) on every forward.
|
||||
- The joint sequence is ``[condition images, ..., target image]`` — the target
|
||||
noise prediction is the *tail* of the output, not the head.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import (
|
||||
CFGParallelMixin,
|
||||
_get_cfg_world_size_or_one,
|
||||
_unwrap,
|
||||
_wrap,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_cfg_group,
|
||||
get_classifier_free_guidance_rank,
|
||||
)
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class QwenImage21CFGParallelMixin(CFGParallelMixin, ProgressBarMixin):
|
||||
"""Base Mixin class for the Qwen-Image 2.1 pipeline providing shared CFG methods."""
|
||||
|
||||
def predict_noise_maybe_with_cfg(
|
||||
self,
|
||||
do_true_cfg: bool,
|
||||
true_cfg_scale: float,
|
||||
positive_kwargs: dict[str, Any],
|
||||
negative_kwargs: dict[str, Any] | None,
|
||||
cfg_normalize: bool = False,
|
||||
output_slice: int | None = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
# The base implementation slices `[:, :output_slice]` per branch, which
|
||||
# matches the 2.0 condition-last layout. 2.1 concatenates condition
|
||||
# latents *before* the target, so the target prediction is the tail of
|
||||
# the joint output. Each branch must be tail-sliced *before* the CFG
|
||||
# combine: positive and negative prompts tokenize to different lengths,
|
||||
# so the full joint outputs do not align positionally.
|
||||
def tail_slice(pred: torch.Tensor | tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, ...]:
|
||||
if output_slice is None:
|
||||
return _wrap(pred)
|
||||
return tuple(p[:, -output_slice:] for p in _wrap(pred))
|
||||
|
||||
if do_true_cfg:
|
||||
if _get_cfg_world_size_or_one() > 1:
|
||||
cfg_group = get_cfg_group()
|
||||
cfg_rank = get_classifier_free_guidance_rank()
|
||||
branch_kwargs = positive_kwargs if cfg_rank == 0 else negative_kwargs
|
||||
local_pred = tail_slice(self.predict_noise(**branch_kwargs))
|
||||
gathered = [cfg_group.all_gather(p, separate_tensors=True) for p in local_pred]
|
||||
positive_noise_pred = tuple(g[0] for g in gathered)
|
||||
negative_noise_pred = tuple(g[1] for g in gathered)
|
||||
else:
|
||||
positive_noise_pred = tail_slice(self.predict_noise(**positive_kwargs))
|
||||
negative_noise_pred = tail_slice(self.predict_noise(**negative_kwargs))
|
||||
return self.combine_cfg_noise(
|
||||
positive_noise_pred,
|
||||
negative_noise_pred,
|
||||
true_cfg_scale,
|
||||
cfg_normalize,
|
||||
**({} if kwargs is None else {"kwargs": kwargs}),
|
||||
)
|
||||
|
||||
pred = self.predict_noise(**positive_kwargs)
|
||||
if output_slice is not None:
|
||||
pred = _unwrap(tail_slice(pred))
|
||||
return pred
|
||||
|
||||
def diffuse(
|
||||
self,
|
||||
prompt_embeds: torch.Tensor,
|
||||
prompt_embeds_mask: torch.Tensor,
|
||||
negative_prompt_embeds: torch.Tensor | None,
|
||||
negative_prompt_embeds_mask: torch.Tensor | None,
|
||||
img_mask: torch.Tensor,
|
||||
negative_img_mask: torch.Tensor | None,
|
||||
latents: torch.Tensor,
|
||||
image_latents: torch.Tensor | None,
|
||||
img_shapes: list,
|
||||
timesteps: torch.Tensor,
|
||||
do_true_cfg: bool,
|
||||
true_cfg_scale: float,
|
||||
attention_kwargs: dict[str, Any] | None = None,
|
||||
additional_transformer_kwargs: dict[str, Any] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Diffusion loop with optional classifier-free guidance.
|
||||
|
||||
Args:
|
||||
prompt_embeds: Positive prompt embeddings
|
||||
prompt_embeds_mask: Mask for positive prompt
|
||||
negative_prompt_embeds: Negative prompt embeddings
|
||||
negative_prompt_embeds_mask: Mask for negative prompt
|
||||
img_mask: Joint-sequence image-slot mask for the positive branch
|
||||
(VLM image slots plus one slot per 2x2 group of target tokens)
|
||||
negative_img_mask: Same for the negative branch
|
||||
latents: Noise latents to denoise (target image only, packed)
|
||||
image_latents: Packed condition-image latents, prepended to the
|
||||
noise latents (default: None for pure text-to-image)
|
||||
img_shapes: Per-sample list of (frame, height, width) latent-token
|
||||
shapes, condition images first and the target last
|
||||
timesteps: Diffusion timesteps
|
||||
do_true_cfg: Whether to apply CFG
|
||||
true_cfg_scale: CFG scale factor
|
||||
attention_kwargs: Passed through to the transformer (attn_path hint)
|
||||
additional_transformer_kwargs: Extra kwargs for the transformer
|
||||
|
||||
Returns:
|
||||
Denoised latents
|
||||
"""
|
||||
# Fresh cache for every generation: the pipeline is a singleton shared
|
||||
# across requests, so cache state must never leak between them.
|
||||
cache_enabled = getattr(self.transformer, "causal_condition", False)
|
||||
kv_cache = [{} for _ in range(len(self.transformer.transformer_blocks))] if cache_enabled else None
|
||||
self.scheduler.set_begin_index(0)
|
||||
self.transformer.do_true_cfg = do_true_cfg
|
||||
additional_transformer_kwargs = additional_transformer_kwargs or {}
|
||||
|
||||
with self.progress_bar(total=len(timesteps)) as pbar:
|
||||
for i, t in enumerate(timesteps):
|
||||
if self.interrupt:
|
||||
continue
|
||||
self._current_timestep = t
|
||||
|
||||
# Broadcast timestep to match batch size
|
||||
timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
|
||||
|
||||
# Condition image latents go in front of the noise latents.
|
||||
latent_model_input = latents
|
||||
if image_latents is not None:
|
||||
latent_model_input = torch.cat([image_latents, latents], dim=1)
|
||||
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep / 1000,
|
||||
"encoder_hidden_states_mask": prompt_embeds_mask,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"img_shapes": img_shapes,
|
||||
"img_mask": img_mask,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"kv_cache": kv_cache,
|
||||
"cache_branch": "cond",
|
||||
**additional_transformer_kwargs,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep / 1000,
|
||||
"encoder_hidden_states_mask": negative_prompt_embeds_mask,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"img_shapes": img_shapes,
|
||||
"img_mask": negative_img_mask,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"kv_cache": kv_cache,
|
||||
"cache_branch": "uncond",
|
||||
**additional_transformer_kwargs,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
# Prefill (first step) returns the full joint sequence and
|
||||
# caches the prefix; decode steps return target tokens only.
|
||||
# Slicing the target tail is correct for both.
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg,
|
||||
true_cfg_scale,
|
||||
positive_kwargs,
|
||||
negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
output_slice=latents.size(1),
|
||||
)
|
||||
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
|
||||
pbar.update()
|
||||
|
||||
return latents
|
||||
@@ -0,0 +1,457 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CUDA Graph capture for Qwen-Image-2.1 KV-cache decode steps.
|
||||
|
||||
Decode (denoising steps 1..N) recomputes only the target image's tokens: the
|
||||
shape is identical every step and the timestep-independent prefix K/V come from
|
||||
the cross-step KV cache. That is the textbook CUDA-graph case, except the
|
||||
default cache stores prefix K/V in freshly allocated tensors, so a captured
|
||||
graph would bind addresses that the next request (or a step-mode cache merge)
|
||||
no longer owns.
|
||||
|
||||
Requests retain ownership of their prefix K/V. Decode refreshes separate
|
||||
fixed-address graph buffers when the active prefix tensors change. Graphs
|
||||
are keyed by branch, batch, prefix length, complete image layout,
|
||||
dtype, device and backend, so equal token counts with different RoPE layouts stay separate.
|
||||
|
||||
Each key admits a single live owner, tracked by a weak reference to the
|
||||
registering request's prefix K tensor. A second in-flight request whose cache
|
||||
aliases the same key is refused graph decode and stays eager until the
|
||||
owner's cache is released, so shared static buffers are never overwritten
|
||||
under a concurrent owner.
|
||||
|
||||
Memory note: graph entries own an additional copy of the prefix K/V. Decode
|
||||
attention concatenates that prefix with target K/V inside the captured region;
|
||||
transient tensors use the graph memory pool.
|
||||
|
||||
The model runner regionally torch.compile's the transformer blocks even when
|
||||
decode graphs are enabled, so capture records the compiled (fused) kernels;
|
||||
inductor's own cudagraphs stay off so the two graph layers never stack.
|
||||
|
||||
Fallbacks (all logged, all eager): CUDA unavailable, model-level CPU offload
|
||||
without persistent weight staging, sequence/ring/tensor parallelism,
|
||||
HSDP/offload hooks, KV-cache quantization, dynamic LoRA wrappers, padded text
|
||||
masks (the masked attention path branches on mask contents, which cannot be
|
||||
captured), unknown graph key at decode, a second live request aliasing an
|
||||
owned graph key, and capture failure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import weakref
|
||||
from collections import OrderedDict
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.models.qwen_image_21.qwen_image_21_transformer import (
|
||||
QwenImage21Transformer2DModel,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _dynamic_lora_wrappers_present(module: torch.nn.Module) -> bool:
|
||||
"""True once ``DiffusionLoRAManager`` has wrapped layers under ``module``.
|
||||
|
||||
A captured graph records the layers and branches that ran at capture time;
|
||||
binding or rescaling an adapter afterwards changes the math without
|
||||
changing the shapes the graph was keyed on, so graphs and dynamic LoRA
|
||||
must not mix. Same reasoning as ``sensenova_u1.paged_decode``.
|
||||
"""
|
||||
try:
|
||||
from vllm.lora.layers import BaseLayerWithLoRA
|
||||
except ImportError: # pragma: no cover - depends on the wheel
|
||||
return False
|
||||
return any(isinstance(m, BaseLayerWithLoRA) for m in module.modules())
|
||||
|
||||
|
||||
class QwenImage21DecodeGraphEntry:
|
||||
"""Static state for one captured decode graph.
|
||||
|
||||
Buffers are created outside inference mode so replay-side ``copy_`` is
|
||||
legal regardless of the caller's grad context. ``graph`` is None until the
|
||||
first decode step captures it. Failed entries are replaced by a ``None``
|
||||
marker in the manager so their buffers can be released.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
branch: str,
|
||||
batch_size: int,
|
||||
prefix_len: int,
|
||||
target_tokens: int,
|
||||
in_channels: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
kv_shapes: list[torch.Size],
|
||||
kv_dtype: torch.dtype,
|
||||
freqs: torch.Tensor,
|
||||
):
|
||||
self.branch = branch
|
||||
self.target_tokens = target_tokens
|
||||
self.prefix_len = prefix_len
|
||||
with torch.inference_mode(False):
|
||||
self.hidden = torch.zeros(batch_size, target_tokens, in_channels, dtype=dtype, device=device)
|
||||
self.timestep = torch.zeros(batch_size, dtype=dtype, device=device)
|
||||
self.k = [torch.zeros(shape, dtype=kv_dtype, device=device) for shape in kv_shapes]
|
||||
self.v = [torch.zeros(shape, dtype=kv_dtype, device=device) for shape in kv_shapes]
|
||||
self.target_token_mask = torch.ones(target_tokens, dtype=torch.bool, device=device)
|
||||
# RoPE frequencies are layout-dependent but step-independent. Clone
|
||||
# into a normal (non-inference) tensor: the prefill runs under
|
||||
# inference_mode, and the captured body takes views of this tensor
|
||||
# outside it, which is forbidden for inference tensors.
|
||||
self.freqs = freqs.clone()
|
||||
self.attn_metadata = None
|
||||
# Per-block cache dicts with the same shape as the legacy protocol.
|
||||
self.block_caches = [{branch: {"key": self.k[i], "value": self.v[i]}} for i in range(len(kv_shapes))]
|
||||
self.graph: torch.cuda.CUDAGraph | None = None
|
||||
self.output: torch.Tensor | None = None
|
||||
self.captures = 0
|
||||
self.prefix_sources: list[tuple[weakref.ReferenceType[torch.Tensor], int | None]] = []
|
||||
|
||||
def refresh_prefix(self, kv_cache: list[dict[str, dict[str, torch.Tensor]]]) -> None:
|
||||
sources = [block[self.branch][name] for block in kv_cache for name in ("key", "value")]
|
||||
versions = [None if tensor.is_inference() else tensor._version for tensor in sources]
|
||||
if len(sources) == len(self.prefix_sources) and all(
|
||||
previous() is tensor and version == current_version
|
||||
for (previous, version), tensor, current_version in zip(self.prefix_sources, sources, versions)
|
||||
):
|
||||
return
|
||||
# Prefill replaces inference-mode tensors; decode treats them as immutable.
|
||||
# Weak references avoid retaining completed requests or mistaking a reused
|
||||
# allocation for the same prefix. Normal tensors also track in-place writes.
|
||||
self.prefix_sources = []
|
||||
for i, block in enumerate(kv_cache):
|
||||
self.k[i].copy_(block[self.branch]["key"])
|
||||
self.v[i].copy_(block[self.branch]["value"])
|
||||
self.prefix_sources = [(weakref.ref(tensor), version) for tensor, version in zip(sources, versions)]
|
||||
|
||||
def capture(self, body) -> None:
|
||||
"""Warm up on a side stream, then capture one decode step.
|
||||
|
||||
Warm-up runs the same computation it will record, allocating cuBLAS
|
||||
workspaces and any lazy buffers outside the capture. Decode writes
|
||||
nothing to the cache (prefix K/V are read-only), so warm-up is
|
||||
idempotent.
|
||||
"""
|
||||
side = torch.cuda.Stream()
|
||||
side.wait_stream(torch.cuda.current_stream())
|
||||
try:
|
||||
with torch.cuda.stream(side), torch.inference_mode(False), torch.no_grad():
|
||||
for _ in range(2):
|
||||
body()
|
||||
finally:
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
# The platform-wide pool is shared with every other captured path in
|
||||
# the tree; entries only keep their *output* alive, whose tiny size
|
||||
# makes reuse safe because the caller clones it before any other graph
|
||||
# on the pool replays.
|
||||
with torch.inference_mode(False), torch.no_grad():
|
||||
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
|
||||
self.output = body()
|
||||
self.graph = graph
|
||||
self.captures += 1
|
||||
|
||||
|
||||
class QwenImage21DecodeGraphManager:
|
||||
"""Owns the static KV buffers and captured graphs of one transformer."""
|
||||
|
||||
def __init__(self, model: QwenImage21Transformer2DModel, max_entries: int = 8, model_level_offload: bool = False):
|
||||
self.model = model
|
||||
self.max_entries = max_entries
|
||||
self.model_level_offload = model_level_offload
|
||||
self.entries: OrderedDict[tuple, QwenImage21DecodeGraphEntry | None] = OrderedDict()
|
||||
# Key -> weak reference to the prefix K tensor of the request that owns
|
||||
# the entry's static buffers. Dead references mean the owning request
|
||||
# finished and its cache was released, so the key is free to adopt.
|
||||
self._owners: dict[tuple, weakref.ReferenceType[torch.Tensor]] = {}
|
||||
self._static_eligible: bool | None = None
|
||||
|
||||
# ── eligibility ──
|
||||
|
||||
def _offload_reason(self) -> str | None:
|
||||
"""Reason decode graphs are incompatible with model-level offload, if any.
|
||||
|
||||
Model-level (sequential) offload registers its hook on the top-level
|
||||
transformer module, not on individual blocks. With persistent staging
|
||||
the hook keeps the DiT on fixed device storage and only rebinds
|
||||
``p.data`` around each generation, and the swap completes in
|
||||
``pre_forward`` before capture or replay begin, so captured weight
|
||||
pointers stay valid. Without staging every swap allocates fresh
|
||||
storage and replay would read stale pointers, so capture stays
|
||||
disabled and decode falls back to eager.
|
||||
"""
|
||||
registry = getattr(self.model, "_hook_registry", None)
|
||||
hooks = registry._hooks if registry is not None else {}
|
||||
if hooks:
|
||||
from vllm_omni.diffusion.offloader.sequential_backend import (
|
||||
SequentialOffloadHook,
|
||||
sequential_offload_staging_active,
|
||||
)
|
||||
|
||||
if set(hooks) <= {SequentialOffloadHook._HOOK_NAME} and sequential_offload_staging_active(self.model):
|
||||
return None
|
||||
return "model module carries offload/cache hooks"
|
||||
if self.model_level_offload:
|
||||
return "model-level CPU offload without persistent staging swaps weight storage after capture"
|
||||
return None
|
||||
|
||||
def _check_static_eligibility(self) -> bool:
|
||||
"""One-time checks that cannot change after weights are loaded."""
|
||||
model = self.model
|
||||
reason = self._offload_reason()
|
||||
param = next(model.parameters(), None)
|
||||
if reason is None and (param is None or param.device.type != "cuda"):
|
||||
reason = f"model is not on CUDA (device={None if param is None else param.device})"
|
||||
elif reason is None:
|
||||
parallel_config = getattr(model, "parallel_config", None)
|
||||
if parallel_config is not None:
|
||||
sp = getattr(parallel_config, "sequence_parallel_size", None) or 1
|
||||
if sp > 1 or getattr(parallel_config, "ring_degree", 1) > 1:
|
||||
reason = "sequence/ring parallelism changes decode shapes per rank"
|
||||
elif getattr(parallel_config, "use_hsdp", False):
|
||||
reason = "HSDP wraps blocks with gather hooks outside the captured region"
|
||||
if reason is None:
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
|
||||
if get_tensor_model_parallel_world_size() > 1:
|
||||
reason = "tensor parallelism runs collectives inside the captured region"
|
||||
if reason is None:
|
||||
for block in model.transformer_blocks:
|
||||
if hasattr(block, "_hook_registry") or hasattr(block, "_omni_original_forward"):
|
||||
reason = "transformer blocks carry offload/cache hooks"
|
||||
break
|
||||
if reason is None and model.transformer_blocks:
|
||||
attn_layer = model.transformer_blocks[0].attn.attn
|
||||
if getattr(attn_layer, "_kv_cache_dtype", None) is not None:
|
||||
reason = "KV-cache quantization is enabled"
|
||||
|
||||
if reason is not None:
|
||||
logger.warning_once("Qwen-Image-2.1 CUDA graph decode disabled: %s. Falling back to eager decode.", reason)
|
||||
return False
|
||||
return True
|
||||
|
||||
def eligible(self) -> bool:
|
||||
if self._static_eligible is None:
|
||||
self._static_eligible = self._check_static_eligibility()
|
||||
return self._static_eligible
|
||||
|
||||
def _backend_name(self) -> str:
|
||||
backend = self.model.transformer_blocks[0].attn.attn.attn_backend
|
||||
return backend.get_name() if backend is not None else "custom"
|
||||
|
||||
# ── prefill registration ──
|
||||
|
||||
def register_prefill(
|
||||
self,
|
||||
*,
|
||||
kv_cache: list[dict[str, dict[str, torch.Tensor]]],
|
||||
cache_branch: str,
|
||||
prefix_len: int,
|
||||
img_shapes: list[tuple[int, int, int]],
|
||||
target_freqs: torch.Tensor,
|
||||
joint_key_valid: torch.Tensor | None,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Prepare graph buffers without changing the request's prefix K/V.
|
||||
|
||||
The first in-flight request to register a key owns its static buffers;
|
||||
registrations for the same key from another live request are refused
|
||||
so their decode stays eager until the owner completes.
|
||||
"""
|
||||
if not self.eligible() or torch.compiler.is_compiling():
|
||||
return
|
||||
if _dynamic_lora_wrappers_present(self.model):
|
||||
if self.entries:
|
||||
logger.warning_once(
|
||||
"Qwen-Image-2.1 CUDA graph decode: LoRA wrappers appeared; "
|
||||
"dropping captured graphs and falling back to eager decode."
|
||||
)
|
||||
self.entries.clear()
|
||||
return
|
||||
if joint_key_valid is not None and not bool(joint_key_valid.all()):
|
||||
# The masked decode path (varlen-unpack or 4D mask) branches on mask
|
||||
# contents and cannot be captured; this request stays eager.
|
||||
logger.warning_once(
|
||||
"Qwen-Image-2.1 CUDA graph decode: padded text mask present; this request falls back to eager decode."
|
||||
)
|
||||
return
|
||||
|
||||
first = kv_cache[0].get(cache_branch)
|
||||
if first is None or "key" not in first:
|
||||
return
|
||||
if any(name.endswith("_scale") for name in first):
|
||||
# Quantized (FP8) prefix storage: the static K/V buffers assume a single
|
||||
# dtype and the captured body would not see the dequant scales (a bf16 K +
|
||||
# fp8 V mix would even copy_ fp8 into bf16 buffers, silently dropping the
|
||||
# scales). Keep quantized-cache requests on the eager decode path.
|
||||
logger.warning_once(
|
||||
"Qwen-Image-2.1 CUDA graph decode: quantized prefix KV cache "
|
||||
"(prefix_kv_cache_dtype) is not graph-capturable; falling back to eager decode."
|
||||
)
|
||||
return
|
||||
batch_size = first["key"].shape[0]
|
||||
key = (
|
||||
cache_branch,
|
||||
batch_size,
|
||||
prefix_len,
|
||||
tuple(tuple(shape) for shape in img_shapes),
|
||||
dtype,
|
||||
first["key"].device.index,
|
||||
self._backend_name(),
|
||||
)
|
||||
|
||||
if key not in self.entries:
|
||||
while len(self.entries) >= self.max_entries:
|
||||
evicted_key, evicted_entry = self.entries.popitem(last=False)
|
||||
self._owners.pop(evicted_key, None)
|
||||
del evicted_entry
|
||||
logger.debug("Evicting decode graph entry %s (max_entries=%d)", evicted_key, self.max_entries)
|
||||
allocation_error = None
|
||||
try:
|
||||
self.entries[key] = QwenImage21DecodeGraphEntry(
|
||||
branch=cache_branch,
|
||||
batch_size=batch_size,
|
||||
prefix_len=prefix_len,
|
||||
target_tokens=target_freqs.shape[0],
|
||||
in_channels=self.model.in_channels,
|
||||
dtype=dtype,
|
||||
device=first["key"].device,
|
||||
kv_shapes=[block_cache[cache_branch]["key"].shape for block_cache in kv_cache],
|
||||
kv_dtype=first["key"].dtype,
|
||||
freqs=target_freqs,
|
||||
)
|
||||
except torch.OutOfMemoryError as exc:
|
||||
allocation_error = str(exc)
|
||||
if allocation_error is not None:
|
||||
# Keep a bounded failure marker without retaining tensors or the exception traceback.
|
||||
self.entries[key] = None
|
||||
current_omni_platform.empty_cache()
|
||||
logger.warning(
|
||||
"Qwen-Image-2.1 CUDA graph allocation failed for key=%s: %s. Falling back to eager decode.",
|
||||
key,
|
||||
allocation_error,
|
||||
)
|
||||
return
|
||||
else:
|
||||
self.entries.move_to_end(key)
|
||||
|
||||
entry = self.entries[key]
|
||||
if entry is not None:
|
||||
owner = self._owners.get(key)
|
||||
owner_prefix = owner() if owner is not None else None
|
||||
if owner_prefix is not None and owner_prefix is not first["key"]:
|
||||
# Another in-flight request still owns this entry's static
|
||||
# buffers; refreshing them would overwrite that request's
|
||||
# prefix mid-generation. This request decodes eagerly until
|
||||
# the owner's cache is released and the weak reference dies.
|
||||
logger.warning_once(
|
||||
"Qwen-Image-2.1 CUDA graph decode: entry key=%s is owned by another in-flight "
|
||||
"request; this request falls back to eager decode.",
|
||||
key,
|
||||
)
|
||||
return
|
||||
self._owners[key] = weakref.ref(first["key"])
|
||||
entry.prefix_sources = []
|
||||
|
||||
logger.debug(
|
||||
"Registered decode graph entry key=%s (entries=%d, captured=%d)",
|
||||
key,
|
||||
len(self.entries),
|
||||
sum(1 for e in self.entries.values() if e is not None and e.graph is not None),
|
||||
)
|
||||
|
||||
# ── decode replay ──
|
||||
|
||||
def try_decode(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
kv_cache: list[dict[str, dict[str, torch.Tensor]]],
|
||||
cache_branch: str,
|
||||
img_shapes: list[tuple[int, int, int]],
|
||||
img_mask: torch.Tensor,
|
||||
encoder_hidden_states_mask: torch.Tensor | None,
|
||||
) -> torch.Tensor | None:
|
||||
"""Replay the captured decode step, or return None to fall back to eager."""
|
||||
if not self.eligible() or torch.compiler.is_compiling():
|
||||
return None
|
||||
if _dynamic_lora_wrappers_present(self.model):
|
||||
self.clear()
|
||||
return None
|
||||
if encoder_hidden_states_mask is not None:
|
||||
text_positions = ~img_mask[0, : encoder_hidden_states_mask.shape[1]]
|
||||
if not bool(encoder_hidden_states_mask[:, text_positions].all()):
|
||||
return None
|
||||
first = kv_cache[0][cache_branch]
|
||||
cached_key = first["key"]
|
||||
if cached_key.device.type != "cuda":
|
||||
# Registration is device-agnostic (unit tests exercise the buffer
|
||||
# machinery on CPU), but capture/replay is CUDA-only: an empty
|
||||
# graph over CPU ops would replay as a stale no-op.
|
||||
return None
|
||||
key = (
|
||||
cache_branch,
|
||||
cached_key.shape[0],
|
||||
cached_key.shape[1],
|
||||
tuple(tuple(shape) for shape in img_shapes),
|
||||
hidden_states.dtype,
|
||||
cached_key.device.index,
|
||||
self._backend_name(),
|
||||
)
|
||||
entry = self.entries.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
self.entries.move_to_end(key)
|
||||
|
||||
owner = self._owners.get(key)
|
||||
owner_prefix = owner() if owner is not None else None
|
||||
if owner_prefix is not None and owner_prefix is not cached_key:
|
||||
# The entry is owned by a different in-flight request (see
|
||||
# register_prefill); this request stays on the eager decode path
|
||||
# until the owner's cache is released.
|
||||
return None
|
||||
|
||||
entry.hidden.copy_(hidden_states[:, -entry.target_tokens :])
|
||||
entry.timestep.copy_(timestep)
|
||||
# Graph buffers are scratch space; request caches must never alias them.
|
||||
entry.refresh_prefix(kv_cache)
|
||||
|
||||
if entry.graph is None:
|
||||
capture_error = None
|
||||
try:
|
||||
entry.capture(lambda: self.model._decode_graph_forward(entry))
|
||||
except Exception as exc:
|
||||
capture_error = str(exc)
|
||||
if capture_error is not None:
|
||||
self.entries[key] = None
|
||||
del entry
|
||||
current_omni_platform.empty_cache()
|
||||
logger.warning(
|
||||
"Qwen-Image-2.1 CUDA graph capture failed for key=%s: %s. "
|
||||
"Falling back to eager decode for this shape.",
|
||||
key,
|
||||
capture_error,
|
||||
)
|
||||
return None
|
||||
logger.info("Captured Qwen-Image-2.1 decode graph for key=%s", key)
|
||||
entry.graph.replay()
|
||||
assert entry.output is not None
|
||||
# Clone before returning: the output lives in the shared graph pool and
|
||||
# the next replay (of this or another graph on the pool) overwrites it.
|
||||
return entry.output.clone()
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Drop all entries, freeing buffers and captured graphs."""
|
||||
self.entries.clear()
|
||||
self._owners.clear()
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user