From 4049f356f21bbd56df879af78f79b40e1f66981c Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Fri, 28 Nov 2025 14:22:11 +0800 Subject: [PATCH] [Diffusion] Qwen image support (#82) Signed-off-by: zjy0516 --- docs/contributing/DOCS_GUIDE.md | 4 +- .../offline_inference/diffusion/qwen_image.py | 20 + .../offline_inference/qwen2_5_omni/end2end.py | 4 +- pyproject.toml | 6 +- vllm_omni/__init__.py | 4 +- vllm_omni/diffusion/__init__.py | 0 vllm_omni/diffusion/data.py | 210 +++++ vllm_omni/diffusion/diffusion_engine.py | 130 +++ vllm_omni/diffusion/distributed/__init__.py | 0 vllm_omni/diffusion/distributed/utils.py | 11 + vllm_omni/diffusion/layers/__init__.py | 0 .../models/qwen_image/pipeline_qwen_image.py | 776 ++++++++++++++++++ .../qwen_image/qwen_image_transformer.py | 700 ++++++++++++++++ vllm_omni/diffusion/omni_diffusion.py | 84 ++ vllm_omni/diffusion/registry.py | 51 ++ vllm_omni/diffusion/request.py | 179 ++++ vllm_omni/diffusion/utils/__init__.py | 0 vllm_omni/diffusion/utils/hf_utils.py | 20 + vllm_omni/diffusion/utils/network_utils.py | 19 + vllm_omni/diffusion/worker/gpu_worker.py | 196 +++++ vllm_omni/entrypoints/omni.py | 27 + vllm_omni/entrypoints/stage_utils.py | 2 +- 22 files changed, 2435 insertions(+), 8 deletions(-) create mode 100644 examples/offline_inference/diffusion/qwen_image.py create mode 100644 vllm_omni/diffusion/__init__.py create mode 100644 vllm_omni/diffusion/data.py create mode 100644 vllm_omni/diffusion/diffusion_engine.py create mode 100644 vllm_omni/diffusion/distributed/__init__.py create mode 100644 vllm_omni/diffusion/distributed/utils.py create mode 100644 vllm_omni/diffusion/layers/__init__.py create mode 100644 vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py create mode 100644 vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py create mode 100644 vllm_omni/diffusion/omni_diffusion.py create mode 100644 vllm_omni/diffusion/registry.py create mode 100644 vllm_omni/diffusion/request.py create mode 100644 vllm_omni/diffusion/utils/__init__.py create mode 100644 vllm_omni/diffusion/utils/hf_utils.py create mode 100644 vllm_omni/diffusion/utils/network_utils.py create mode 100644 vllm_omni/diffusion/worker/gpu_worker.py create mode 100644 vllm_omni/entrypoints/omni.py diff --git a/docs/contributing/DOCS_GUIDE.md b/docs/contributing/DOCS_GUIDE.md index ccba0663f..f4c4265ab 100644 --- a/docs/contributing/DOCS_GUIDE.md +++ b/docs/contributing/DOCS_GUIDE.md @@ -38,7 +38,7 @@ The documentation automatically extracts docstrings from the code using mkdocstr Example docstring: ```python -class OmniLLM: +class Omni: """Main entry point for vLLM-omni inference. This class provides a high-level interface for running multi-modal @@ -50,7 +50,7 @@ class OmniLLM: **kwargs: Additional arguments passed to the engine Example: - >>> llm = OmniLLM(model="Qwen/Qwen2.5-Omni") + >>> llm = Omni(model="Qwen/Qwen2.5-Omni") >>> outputs = llm.generate(prompts="Hello") """ ``` diff --git a/examples/offline_inference/diffusion/qwen_image.py b/examples/offline_inference/diffusion/qwen_image.py new file mode 100644 index 000000000..24c43ca02 --- /dev/null +++ b/examples/offline_inference/diffusion/qwen_image.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm_omni.entrypoints.omni import Omni + +model_name = "Qwen/Qwen-Image" +prompt = "a cup of coffee on the table" +if __name__ == "__main__": + m = Omni( + model=model_name, + ) + image = m.generate( + prompt, + height=1024, + width=1024, + generator=torch.Generator(device="cuda").manual_seed(42), + ) + image[0].save("qwen_image_output.png") diff --git a/examples/offline_inference/qwen2_5_omni/end2end.py b/examples/offline_inference/qwen2_5_omni/end2end.py index be4e2c1a1..9045172ae 100644 --- a/examples/offline_inference/qwen2_5_omni/end2end.py +++ b/examples/offline_inference/qwen2_5_omni/end2end.py @@ -16,7 +16,7 @@ from vllm.multimodal.image import convert_image_mode from vllm.sampling_params import SamplingParams from vllm.utils import FlexibleArgumentParser -from vllm_omni import OmniLLM +from vllm_omni import Omni SEED = 42 @@ -140,7 +140,7 @@ def main(args): model_name = "Qwen/Qwen2.5-Omni-7B" query_result = query_map[args.query_type]() - omni_llm = OmniLLM( + omni_llm = Omni( model=model_name, log_stats=args.enable_stats, log_file=("omni_llm_pipeline.log" if args.enable_stats else None), diff --git a/pyproject.toml b/pyproject.toml index 513ee1317..498c959f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,8 @@ dependencies = [ # "decord>=0.6.0", # install manually "librosa>=0.11.0", "resampy>=0.4.3", + "diffusers==0.35.2", + "accelerate==1.12.0", # "vllm==0.11.0", # TODO: fix the entrypoints overwrite problem ] @@ -160,5 +162,7 @@ extend-ignore-identifiers-re = [ ".*_thw", ".*thw", "ein", - ".*arange" + ".*arange", + ".*MoBA", + ".*temperal_downsample" ] diff --git a/vllm_omni/__init__.py b/vllm_omni/__init__.py index adf9efc32..be1afdf00 100644 --- a/vllm_omni/__init__.py +++ b/vllm_omni/__init__.py @@ -23,7 +23,7 @@ from .config import OmniModelConfig from .entrypoints.async_omni_llm import AsyncOmniLLM # Main entry points -from .entrypoints.omni_llm import OmniLLM +from .entrypoints.omni import Omni from .version import __version__, __version_tuple__ # isort:skip @@ -32,7 +32,7 @@ __all__ = [ "__version__", "__version_tuple__", # Main components - "OmniLLM", + "Omni", "AsyncOmniLLM", # Configuration "OmniModelConfig", diff --git a/vllm_omni/diffusion/__init__.py b/vllm_omni/diffusion/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py new file mode 100644 index 000000000..1b612264b --- /dev/null +++ b/vllm_omni/diffusion/data.py @@ -0,0 +1,210 @@ +# adapted from sglang and fastvideo +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import enum +import random +from dataclasses import dataclass, field +from typing import Any, Callable + +import torch +from vllm.logger import init_logger + +from vllm_omni.diffusion.utils.network_utils import is_port_available + +logger = init_logger(__name__) + + +@dataclass +class OmniDiffusionConfig: + # Model and path configuration (for convenience) + model: str + + model_class_name: str | None = None + + # Attention + # attention_backend: str = None + + # Running mode + # mode: ExecutionMode = ExecutionMode.INFERENCE + + # Workload type + # workload_type: WorkloadType = WorkloadType.T2V + + # Cache strategy + cache_strategy: str = "none" + + # Distributed executor backend + distributed_executor_backend: str = "mp" + nccl_port: int | None = None + + # HuggingFace specific parameters + trust_remote_code: bool = False + revision: str | None = None + + # Parallelism + num_gpus: int = 1 + tp_size: int = -1 + sp_degree: int = -1 + # sequence parallelism + ulysses_degree: int | None = None + ring_degree: int | None = None + # data parallelism + # number of data parallelism groups + dp_size: int = 1 + # number of gpu in a dp group + dp_degree: int = 1 + # cfg parallel + enable_cfg_parallel: bool = False + + hsdp_replicate_dim: int = 1 + hsdp_shard_dim: int = -1 + dist_timeout: int | None = None # timeout for torch.distributed + + # pipeline_config: PipelineConfig = field(default_factory=PipelineConfig, repr=False) + + # LoRA parameters + # (Wenxuan) prefer to keep it here instead of in pipeline config to not make it complicated. + lora_path: str | None = None + lora_nickname: str = "default" # for swapping adapters in the pipeline + # can restrict layers to adapt, e.g. ["q_proj"] + # Will adapt only q, k, v, o by default. + lora_target_modules: list[str] | None = None + + output_type: str = "pil" + + # CPU offload parameters + dit_cpu_offload: bool = True + use_fsdp_inference: bool = False + text_encoder_cpu_offload: bool = True + image_encoder_cpu_offload: bool = True + vae_cpu_offload: bool = True + pin_cpu_memory: bool = True + + # STA (Sliding Tile Attention) parameters + mask_strategy_file_path: str | None = None + # STA_mode: STA_Mode = STA_Mode.STA_INFERENCE + skip_time_steps: int = 15 + + # Compilation + enable_torch_compile: bool = False + + disable_autocast: bool = False + + # VSA parameters + VSA_sparsity: float = 0.0 # inference/validation sparsity + + # V-MoBA parameters + moba_config_path: str | None = None + # moba_config: dict[str, Any] = field(default_factory=dict) + + # Master port for distributed inference + # TODO: do not hard code + master_port: int | None = None + + # http server endpoint config, would be ignored in local mode + host: str | None = None + port: int | None = None + + scheduler_port: int = 5555 + + # Stage verification + enable_stage_verification: bool = True + + # Prompt text file for batch processing + prompt_file_path: str | None = None + + # model paths for correct deallocation + model_paths: dict[str, str] = field(default_factory=dict) + model_loaded: dict[str, bool] = field( + default_factory=lambda: { + "transformer": True, + "vae": True, + } + ) + override_transformer_cls_name: str | None = None + + # # DMD parameters + # dmd_denoising_steps: List[int] | None = field(default=None) + + # MoE parameters used by Wan2.2 + boundary_ratio: float | None = None + + # Logging + log_level: str = "info" + + def settle_port(self, port: int, port_inc: int = 42, max_attempts: int = 100) -> int: + """ + Find an available port with retry logic. + + Args: + port: Initial port to check + port_inc: Port increment for each attempt + max_attempts: Maximum number of attempts to find an available port + + Returns: + An available port number + + Raises: + RuntimeError: If no available port is found after max_attempts + """ + attempts = 0 + original_port = port + + while attempts < max_attempts: + if is_port_available(port): + if attempts > 0: + logger.info(f"Port {original_port} was unavailable, using port {port} instead") + return port + + attempts += 1 + if port < 60000: + port += port_inc + else: + # Wrap around with randomization to avoid collision + port = 5000 + random.randint(0, 1000) + + raise RuntimeError( + f"Failed to find available port after {max_attempts} attempts (started from port {original_port})" + ) + + def __post_init__(self): + # TODO: remove hard code + initial_master_port = (self.master_port or 30005) + random.randint(0, 100) + self.master_port = self.settle_port(initial_master_port, 37) + + @classmethod + def from_kwargs(cls, **kwargs: Any) -> "OmniDiffusionConfig": + return cls(**kwargs) + + +@dataclass +class DiffusionOutput: + """ + Final output (after pipeline completion) + """ + + output: torch.Tensor | None = None + trajectory_timesteps: list[torch.Tensor] | None = None + trajectory_latents: torch.Tensor | None = None + trajectory_decoded: list[torch.Tensor] | None = None + error: str | None = None + + post_process_func: Callable[..., Any] | None = None + + # logged timings info, directly from Req.timings + # timings: Optional["RequestTimings"] = None + + +class AttentionBackendEnum(enum.Enum): + FA = enum.auto() + SLIDING_TILE_ATTN = enum.auto() + TORCH_SDPA = enum.auto() + SAGE_ATTN = enum.auto() + SAGE_ATTN_THREE = enum.auto() + VIDEO_SPARSE_ATTN = enum.auto() + VMOBA_ATTN = enum.auto() + AITER = enum.auto() + NO_ATTENTION = enum.auto() + + def __str__(self): + return self.name.lower() diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py new file mode 100644 index 000000000..cd1e20458 --- /dev/null +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import multiprocessing as mp + +from vllm.logger import init_logger + +from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.diffusion.registry import get_diffusion_post_process_func +from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.diffusion.scheduler import scheduler +from vllm_omni.diffusion.worker.gpu_worker import WorkerProc + +logger = init_logger(__name__) + + +class DiffusionEngine: + """The diffusion engine for vLLM-Omni diffusion models.""" + + def __init__(self, od_config: OmniDiffusionConfig): + """Initialize the diffusion engine. + + Args: + config: The configuration for the diffusion engine. + """ + self.od_config = od_config + + self.post_process_func = get_diffusion_post_process_func(od_config) + + self._make_client() + + def step(self, requests: list[OmniDiffusionRequest]): + try: + output = self.add_req_and_wait_for_response(requests) + if output.error: + raise Exception(f"{output.error}") + logger.info("Generation completed successfully.") + return self.post_process_func(output.output) + except Exception as e: + logger.error(f"Generation failed: {e}") + return None + + @staticmethod + def make_engine(config: OmniDiffusionConfig) -> "DiffusionEngine": + """Factory method to create a DiffusionEngine instance. + + Args: + config: The configuration for the diffusion engine. + + Returns: + An instance of DiffusionEngine. + """ + return DiffusionEngine(config) + + def _make_client(self): + # TODO rename it + scheduler.initialize(self.od_config) + + # Get the broadcast handle from the initialized scheduler + broadcast_handle = scheduler.get_broadcast_handle() + + processes, result_handle = self._launch_workers( + broadcast_handle=broadcast_handle, + ) + + if result_handle is not None: + scheduler.initialize_result_queue(result_handle) + else: + logger.error("Failed to get result queue handle from workers") + + def _launch_workers(self, broadcast_handle): + od_config = self.od_config + logger.info("Starting server...") + + num_gpus = od_config.num_gpus + mp.set_start_method("spawn", force=True) + processes = [] + + # Launch all worker processes + scheduler_pipe_readers = [] + scheduler_pipe_writers = [] + + for i in range(num_gpus): + reader, writer = mp.Pipe(duplex=False) + scheduler_pipe_writers.append(writer) + process = mp.Process( + target=WorkerProc.worker_main, + args=( + i, # rank + od_config, + writer, + broadcast_handle, + ), + name=f"DiffusionWorker-{i}", + daemon=True, + ) + scheduler_pipe_readers.append(reader) + process.start() + processes.append(process) + + # Wait for all workers to be ready + scheduler_infos = [] + result_handle = None + for writer in scheduler_pipe_writers: + writer.close() + + for i, reader in enumerate(scheduler_pipe_readers): + try: + data = reader.recv() + except EOFError: + logger.error(f"Rank {i} scheduler is dead. Please check if there are relevant logs.") + processes[i].join() + logger.error(f"Exit code: {processes[i].exitcode}") + raise + + if data["status"] != "ready": + raise RuntimeError("Initialization failed. Please see the error messages above.") + + if i == 0: + result_handle = data.get("result_handle") + + scheduler_infos.append(data) + reader.close() + + logger.debug("All workers are ready") + + return processes, result_handle + + def add_req_and_wait_for_response(self, requests: list[OmniDiffusionRequest]): + return scheduler.add_req(requests) diff --git a/vllm_omni/diffusion/distributed/__init__.py b/vllm_omni/diffusion/distributed/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_omni/diffusion/distributed/utils.py b/vllm_omni/diffusion/distributed/utils.py new file mode 100644 index 000000000..fb930f21a --- /dev/null +++ b/vllm_omni/diffusion/distributed/utils.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import os + +import torch + + +def get_local_device() -> torch.device: + """Return the torch device for the current rank.""" + return torch.device(f"cuda:{os.environ.get('LOCAL_RANK', 0)}") diff --git a/vllm_omni/diffusion/layers/__init__.py b/vllm_omni/diffusion/layers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py new file mode 100644 index 000000000..8aa2e5e44 --- /dev/null +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py @@ -0,0 +1,776 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import inspect +import json +import logging +import math +import os +from typing import Any, Optional, Union + +import numpy as np +import torch +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.autoencoders.autoencoder_kl_qwenimage import ( + AutoencoderKLQwenImage, +) +from diffusers.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from diffusers.utils.torch_utils import randn_tensor +from torch import nn +from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer + +from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.utils import get_local_device +from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import ( + QwenImageTransformer2DModel, +) +from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.model_executor.model_loader.weight_utils import ( + download_weights_from_hf_specific, +) + +logger = logging.getLogger(__name__) + + +def get_qwen_image_post_process_func( + od_config: OmniDiffusionConfig, +): + model_name = od_config.model + model_path = download_weights_from_hf_specific(model_name, None, ["*"]) + vae_config_path = os.path.join(model_path, "vae/config.json") + with open(vae_config_path) as f: + vae_config = json.load(f) + vae_scale_factor = 2 ** len(vae_config["temporal_downsample"]) if "temporal_downsample" in vae_config else 8 + + image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2) + + def post_process_func( + images: torch.Tensor, + ): + return image_processor.postprocess(images) + + return post_process_func + + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[list[int]] = None, + sigmas: Optional[list[float]] = None, + **kwargs, +) -> tuple[torch.Tensor, int]: + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`list[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`list[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def get_timestep_embedding( + timesteps: torch.Tensor, + embedding_dim: int, + flip_sin_to_cos: bool = False, + downscale_freq_shift: float = 1, + scale: float = 1, + max_period: int = 10000, +) -> torch.Tensor: + """ + This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings. + + Args + timesteps (torch.Tensor): + a 1-D Tensor of N indices, one per batch element. These may be fractional. + embedding_dim (int): + the dimension of the output. + flip_sin_to_cos (bool): + Whether the embedding order should be `cos, sin` (if True) or `sin, cos` (if False) + downscale_freq_shift (float): + Controls the delta between frequencies between dimensions + scale (float): + Scaling factor applied to the embeddings. + max_period (int): + Controls the maximum frequency of the embeddings + Returns + torch.Tensor: an [N x dim] Tensor of positional embeddings. + """ + assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array" + + half_dim = embedding_dim // 2 + exponent = -math.log(max_period) * torch.arange(start=0, end=half_dim, dtype=torch.float32, device=timesteps.device) + exponent = exponent / (half_dim - downscale_freq_shift) + + emb = torch.exp(exponent).to(timesteps.dtype) + emb = timesteps[:, None].float() * emb[None, :] + + # scale embeddings + emb = scale * emb + + # concat sine and cosine embeddings + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1) + + # flip sine and cosine embeddings + if flip_sin_to_cos: + emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1) + + # zero pad + if embedding_dim % 2 == 1: + emb = torch.nn.functional.pad(emb, (0, 1, 0, 0)) + return emb + + +def apply_rotary_emb_qwen( + x: torch.Tensor, + freqs_cis: Union[torch.Tensor, tuple[torch.Tensor]], + use_real: bool = True, + use_real_unbind_dim: int = -1, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings + to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are + reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting + tensors contain rotary embeddings and are returned as real tensors. + + Args: + x (`torch.Tensor`): + Query or key tensor to apply rotary embeddings. [B, S, H, D] xk (torch.Tensor): Key tensor to apply + freqs_cis (`tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],) + + Returns: + tuple[torch.Tensor, torch.Tensor]: tuple of modified query tensor and key tensor with rotary embeddings. + """ + if use_real: + cos, sin = freqs_cis # [S, D] + cos = cos[None, None] + sin = sin[None, None] + cos, sin = cos.to(x.device), sin.to(x.device) + + if use_real_unbind_dim == -1: + # Used for flux, cogvideox, hunyuan-dit + x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) # [B, S, H, D//2] + x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3) + elif use_real_unbind_dim == -2: + # Used for Stable Audio, OmniGen, CogView4 and Cosmos + x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2) # [B, S, H, D//2] + x_rotated = torch.cat([-x_imag, x_real], dim=-1) + else: + raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.") + + out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + + return out + else: + x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2)) + freqs_cis = freqs_cis.unsqueeze(1) + x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3) + + return x_out.type_as(x) + + +class QwenImagePipeline( + nn.Module, +): + def __init__( + self, + *, + od_config: OmniDiffusionConfig = None, + prefix: str = "", + ): + super().__init__() + self.device = get_local_device() + # self.hf_config = vllm_config.model_config.hf_config + # TODO support user custom folders + self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained("Qwen/Qwen-Image", subfolder="scheduler") + logger.info("Loaded Qwen-Image scheduler successfully") + from vllm_omni.diffusion.utils.torch_utils import ( + set_default_torch_dtype, + ) # to avoid ci error + + with set_default_torch_dtype(torch.bfloat16): + self.text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained( + "Qwen/Qwen-Image", subfolder="text_encoder" + ) + logger.info("Loaded Qwen-Image text encoder successfully") + self.vae = AutoencoderKLQwenImage.from_pretrained("Qwen/Qwen-Image", subfolder="vae").to(self.device) + logger.info("Loaded Qwen-Image VAE successfully") + self.transformer = QwenImageTransformer2DModel() + logger.info("Initialized Qwen-Image transformer successfully.") + self.tokenizer = Qwen2Tokenizer.from_pretrained("Qwen/Qwen-Image", subfolder="tokenizer") + logger.info("Loaded Qwen-Image tokenizer successfully.") + + self.stage = None + + self.vae_scale_factor = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8 + # QwenImage latents are turned into 2x2 patches and packed. + # This means the latent width and height has to be divisible + # by the patch size. So the vae scale factor is multiplied by the patch size to account for this + # self.image_processor = VaeImageProcessor( + # vae_scale_factor=self.vae_scale_factor * 2 + # ) + self.tokenizer_max_length = 1024 + self.prompt_template_encode = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" # noqa: E501 + self.prompt_template_encode_start_idx = 34 + self.default_sample_size = 128 + + def check_inputs( + self, + prompt, + height, + width, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + prompt_embeds_mask=None, + negative_prompt_embeds_mask=None, + callback_on_step_end_tensor_inputs=None, + max_sequence_length=None, + ): + if height % (self.vae_scale_factor * 2) != 0 or width % (self.vae_scale_factor * 2) != 0: + logger.warning( + f"`height` and `width` have to be divisible by {self.vae_scale_factor * 2} " + "but are {height} and {width}. Dimensions will be resized accordingly" + ) + + # if callback_on_step_end_tensor_inputs is not None and not all( + # k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + # ): + # raise ValueError( + # f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, + # but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + # ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and prompt_embeds_mask is None: + raise ValueError( + "If `prompt_embeds` are provided, `prompt_embeds_mask` also have to be passed. " + "Make sure to generate `prompt_embeds_mask` from the same text encoder " + "that was used to generate `prompt_embeds`." + ) + if negative_prompt_embeds is not None and negative_prompt_embeds_mask is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_prompt_embeds_mask` also have to be passed. " + "Make sure to generate `negative_prompt_embeds_mask` from the same text encoder " + "that was used to generate `negative_prompt_embeds`." + ) + + if max_sequence_length is not None and max_sequence_length > 1024: + raise ValueError(f"`max_sequence_length` cannot be greater than 1024 but is {max_sequence_length}") + + def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + + return split_result + + def _get_qwen_prompt_embeds( + self, + prompt: Union[str, list[str]] = None, + dtype: Optional[torch.dtype] = None, + ): + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + + template = self.prompt_template_encode + drop_idx = self.prompt_template_encode_start_idx + txt = [template.format(e) for e in prompt] + txt_tokens = self.tokenizer( + txt, + max_length=self.tokenizer_max_length + drop_idx, + padding=True, + truncation=True, + return_tensors="pt", + ).to(self.device) + # print(f"attention mask: {txt_tokens.attention_mask}") + encoder_hidden_states = self.text_encoder( + input_ids=txt_tokens.input_ids, + attention_mask=txt_tokens.attention_mask, + output_hidden_states=True, + ) + hidden_states = encoder_hidden_states.hidden_states[-1] + split_hidden_states = self._extract_masked_hidden(hidden_states, txt_tokens.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=dtype) + + return prompt_embeds, encoder_attention_mask + + def encode_prompt( + self, + prompt: Union[str, list[str]], + num_images_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + prompt_embeds_mask: Optional[torch.Tensor] = None, + max_sequence_length: int = 1024, + ): + r""" + + Args: + prompt (`str` or `list[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + """ + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) if prompt_embeds is None else prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds, prompt_embeds_mask = self._get_qwen_prompt_embeds(prompt) + + prompt_embeds = prompt_embeds[:, :max_sequence_length] + prompt_embeds_mask = prompt_embeds_mask[:, :max_sequence_length] + + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1) + prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len) + + return prompt_embeds, prompt_embeds_mask + + @staticmethod + def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + + return latents + + @staticmethod + def _unpack_latents(latents, height, width, vae_scale_factor): + batch_size, num_patches, channels = latents.shape + + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (vae_scale_factor * 2)) + width = 2 * (int(width) // (vae_scale_factor * 2)) + + latents = latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2) + latents = latents.permute(0, 3, 1, 4, 2, 5) + + latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width) + + return latents + + def prepare_latents( + self, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + ) -> torch.Tensor: + # generator=torch.Generator(device="cuda").manual_seed(42) + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (self.vae_scale_factor * 2)) + width = 2 * (int(width) // (self.vae_scale_factor * 2)) + + shape = (batch_size, 1, num_channels_latents, height, width) + + if latents is not None: + return latents.to(device=device, dtype=dtype) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + latents = self._pack_latents(latents, batch_size, num_channels_latents, height, width) + + return latents + + def prepare_timesteps(self, num_inference_steps, sigmas, image_seq_len): + sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas + # image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + self.scheduler.config.get("base_image_seq_len", 256), + self.scheduler.config.get("max_image_seq_len", 4096), + self.scheduler.config.get("base_shift", 0.5), + self.scheduler.config.get("max_shift", 1.15), + ) + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + sigmas=sigmas, + mu=mu, + ) + return timesteps, num_inference_steps + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def current_timestep(self): + return self._current_timestep + + @property + def interrupt(self): + return self._interrupt + + def diffuse( + self, + prompt_embeds, + prompt_embeds_mask, + negative_prompt_embeds, + negative_prompt_embeds_mask, + latents, + img_shapes, + txt_seq_lens, + negative_txt_seq_lens, + timesteps, + do_true_cfg, + guidance, + true_cfg_scale, + ): + self.scheduler.set_begin_index(0) + for i, t in enumerate(timesteps): + if self.interrupt: + continue + self._current_timestep = t + # broadcast to batch dimension and place on same device/dtype as latents + timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype) + noise_pred = self.transformer( + hidden_states=latents, + timestep=timestep / 1000, + guidance=guidance, + encoder_hidden_states_mask=prompt_embeds_mask, + encoder_hidden_states=prompt_embeds, + img_shapes=img_shapes, + txt_seq_lens=txt_seq_lens, + attention_kwargs=self.attention_kwargs, + return_dict=False, + )[0] + if do_true_cfg: + neg_noise_pred = self.transformer( + hidden_states=latents, + timestep=timestep / 1000, + guidance=guidance, + encoder_hidden_states_mask=negative_prompt_embeds_mask, + encoder_hidden_states=negative_prompt_embeds, + img_shapes=img_shapes, + txt_seq_lens=negative_txt_seq_lens, + attention_kwargs=self.attention_kwargs, + return_dict=False, + )[0] + comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) + noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) + noise_pred = comb_pred * (cond_norm / noise_norm) + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + return latents + + def forward( + self, + req: OmniDiffusionRequest, + prompt: Union[str, list[str]] = "", + negative_prompt: Union[str, list[str]] = "", + true_cfg_scale: float = 4.0, + height: int | None = None, + width: int | None = None, + num_inference_steps: int = 50, + sigmas: Optional[list[float]] = None, + guidance_scale: float = 1.0, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, list[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + prompt_embeds: Optional[torch.Tensor] = None, + prompt_embeds_mask: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds_mask: Optional[torch.Tensor] = None, + output_type: Optional[str] = "pil", + attention_kwargs: Optional[dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: list[str] = ["latents"], + max_sequence_length: int = 512, + ) -> DiffusionOutput: + # # TODO: only support single prompt now + # if req.prompt is not None: + # prompt = req.prompt[0] if isinstance(req.prompt, list) else req.prompt + prompt = req.prompt if req.prompt is not None else prompt + negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt + height = req.height or self.default_sample_size * self.vae_scale_factor + width = req.width or self.default_sample_size * self.vae_scale_factor + num_inference_steps = req.num_inference_steps or num_inference_steps + generator = req.generator or generator + true_cfg_scale = req.true_cfg_scale or true_cfg_scale + # 1. check inputs + # 2. encode prompts + # 3. prepare latents and timesteps + # 4. diffusion process + # 5. decode latents + # 6. post-process outputs + self.check_inputs( + prompt, + height, + width, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + prompt_embeds_mask, + negative_prompt_embeds_mask, + callback_on_step_end_tensor_inputs, + max_sequence_length, + ) + + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._current_timestep = None + self._interrupt = False + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + has_neg_prompt = negative_prompt is not None or ( + negative_prompt_embeds is not None and negative_prompt_embeds_mask is not None + ) + + do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + prompt_embeds, prompt_embeds_mask = self.encode_prompt( + prompt=prompt, + prompt_embeds=prompt_embeds, + prompt_embeds_mask=prompt_embeds_mask, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + ) + if do_true_cfg: + negative_prompt_embeds, negative_prompt_embeds_mask = self.encode_prompt( + prompt=negative_prompt, + prompt_embeds=negative_prompt_embeds, + prompt_embeds_mask=negative_prompt_embeds_mask, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + ) + + num_channels_latents = self.transformer.in_channels // 4 + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + self.device, + generator, + latents, + ) + img_shapes = [ + [ + ( + 1, + height // self.vae_scale_factor // 2, + width // self.vae_scale_factor // 2, + ) + ] + ] * batch_size + + timesteps, num_inference_steps = self.prepare_timesteps(num_inference_steps, sigmas, latents.shape[1]) + # num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self._num_timesteps = len(timesteps) + + # handle guidance + if self.transformer.guidance_embeds: + guidance = torch.full([1], guidance_scale, dtype=torch.float32) + guidance = guidance.expand(latents.shape[0]) + else: + guidance = None + + if self.attention_kwargs is None: + self._attention_kwargs = {} + + txt_seq_lens = prompt_embeds_mask.sum(dim=1).tolist() if prompt_embeds_mask is not None else None + negative_txt_seq_lens = ( + negative_prompt_embeds_mask.sum(dim=1).tolist() if negative_prompt_embeds_mask is not None else None + ) + # print inputp params + + latents = self.diffuse( + prompt_embeds, + prompt_embeds_mask, + negative_prompt_embeds, + negative_prompt_embeds_mask, + latents, + img_shapes, + txt_seq_lens, + negative_txt_seq_lens, + timesteps, + do_true_cfg, + guidance, + true_cfg_scale, + ) + + self._current_timestep = None + if output_type == "latent": + image = latents + else: + latents = self._unpack_latents(latents, height, width, self.vae_scale_factor) + latents = latents.to(self.vae.dtype) + latents_mean = ( + torch.tensor(self.vae.config.latents_mean) + .view(1, self.vae.config.z_dim, 1, 1, 1) + .to(latents.device, latents.dtype) + ) + latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to( + latents.device, latents.dtype + ) + latents = latents / latents_std + latents_mean + image = self.vae.decode(latents, return_dict=False)[0][:, :, 0] + # processed_image = self.image_processor.postprocess(image, output_type=output_type) + + return DiffusionOutput(output=image) + + def load_weights(self): + self.load_transformer() + + # TODO: find a better way to load + def load_transformer(self): + import glob + import os + + # Define the weight iterator + def weight_iterator(transformer_path): + if not os.path.exists(transformer_path): + logger.warning(f"Path {transformer_path} does not exist.") + return + + # Look for safetensors first + safetensors_files = glob.glob(os.path.join(transformer_path, "*.safetensors")) + if safetensors_files: + try: + from safetensors.torch import load_file + except ImportError: + logger.warning("safetensors not installed, cannot load .safetensors files.") + return + + for file_path in safetensors_files: + state_dict = load_file(file_path) + for name, tensor in state_dict.items(): + yield name, tensor + else: + # Fallback to bin + bin_files = glob.glob(os.path.join(transformer_path, "*.bin")) + for file_path in bin_files: + state_dict = torch.load(file_path) + for name, tensor in state_dict.items(): + yield name, tensor + + model_name = "Qwen/Qwen-Image" + try: + # Use the arguments as verified by the user + model_path = download_weights_from_hf_specific(model_name, None, ["*"]) + transformer_path = os.path.join(model_path, "transformer") + self.transformer.load_weights(weight_iterator(transformer_path)) + + except Exception as e: + print(f"An error occurred: {e}") + raise e diff --git a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py new file mode 100644 index 000000000..c0ef5fd12 --- /dev/null +++ b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py @@ -0,0 +1,700 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import functools +from collections.abc import Iterable +from typing import Any, Optional, Union + +import torch +import torch.nn as nn +from diffusers.models.attention import FeedForward +from diffusers.models.attention_dispatch import dispatch_attention_fn + +# TODO replace this with vLLM implementation +from diffusers.models.embeddings import TimestepEmbedding, Timesteps +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.normalization import AdaLayerNormContinuous +from vllm.logger import init_logger +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import QKVParallelLinear, ReplicatedLinear + +logger = init_logger(__name__) + + +def apply_rotary_emb_qwen( + x: torch.Tensor, + freqs_cis: Union[torch.Tensor, tuple[torch.Tensor]], + use_real: bool = True, + use_real_unbind_dim: int = -1, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings + to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are + reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting + tensors contain rotary embeddings and are returned as real tensors. + + Args: + x (`torch.Tensor`): + Query or key tensor to apply rotary embeddings. [B, S, H, D] xk (torch.Tensor): Key tensor to apply + freqs_cis (`tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],) + + Returns: + tuple[torch.Tensor, torch.Tensor]: tuple of modified query tensor and key tensor with rotary embeddings. + """ + if use_real: + cos, sin = freqs_cis # [S, D] + cos = cos[None, None] + sin = sin[None, None] + cos, sin = cos.to(x.device), sin.to(x.device) + + if use_real_unbind_dim == -1: + # Used for flux, cogvideox, hunyuan-dit + x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) # [B, S, H, D//2] + x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3) + elif use_real_unbind_dim == -2: + # Used for Stable Audio, OmniGen, CogView4 and Cosmos + x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2) # [B, S, H, D//2] + x_rotated = torch.cat([-x_imag, x_real], dim=-1) + else: + raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.") + + out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + + return out + else: + x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2)) + freqs_cis = freqs_cis.unsqueeze(1) + x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3) + + return x_out.type_as(x) + + +class QwenTimestepProjEmbeddings(nn.Module): + def __init__(self, embedding_dim): + super().__init__() + + self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0, scale=1000) + self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim) + + def forward(self, timestep, hidden_states): + timesteps_proj = self.time_proj(timestep) + timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_states.dtype)) # (N, D) + + conditioning = timesteps_emb + + return conditioning + + +class QwenEmbedRope(nn.Module): + def __init__(self, theta: int, axes_dim: list[int], scale_rope=False): + super().__init__() + self.theta = theta + self.axes_dim = axes_dim + pos_index = torch.arange(4096) + neg_index = torch.arange(4096).flip(0) * -1 - 1 + self.pos_freqs = torch.cat( + [ + self.rope_params(pos_index, self.axes_dim[0], self.theta), + self.rope_params(pos_index, self.axes_dim[1], self.theta), + self.rope_params(pos_index, self.axes_dim[2], self.theta), + ], + dim=1, + ) + self.neg_freqs = torch.cat( + [ + self.rope_params(neg_index, self.axes_dim[0], self.theta), + self.rope_params(neg_index, self.axes_dim[1], self.theta), + self.rope_params(neg_index, self.axes_dim[2], self.theta), + ], + dim=1, + ) + self.rope_cache = {} + + # DO NOT USING REGISTER BUFFER HERE, IT WILL CAUSE COMPLEX NUMBERS LOSE ITS IMAGINARY PART + self.scale_rope = scale_rope + + def rope_params(self, index, dim, theta=10000): + """ + Args: + index: [0, 1, 2, 3] 1D Tensor representing the position index of the token + """ + assert dim % 2 == 0 + freqs = torch.outer( + index, + 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float32).div(dim)), + ) + freqs = torch.polar(torch.ones_like(freqs), freqs) + return freqs + + def forward(self, video_fhw, txt_seq_lens, device): + """ + Args: video_fhw: [frame, height, width] a list of 3 integers representing the shape of the video Args: + txt_length: [bs] a list of 1 integers representing the length of the text + """ + if self.pos_freqs.device != device: + self.pos_freqs = self.pos_freqs.to(device) + self.neg_freqs = self.neg_freqs.to(device) + + if isinstance(video_fhw, list): + video_fhw = video_fhw[0] + if not isinstance(video_fhw, list): + video_fhw = [video_fhw] + + vid_freqs = [] + max_vid_index = 0 + for idx, fhw in enumerate(video_fhw): + frame, height, width = fhw + rope_key = f"{idx}_{height}_{width}" + + if not torch.compiler.is_compiling(): + if rope_key not in self.rope_cache: + self.rope_cache[rope_key] = self._compute_video_freqs(frame, height, width, idx) + video_freq = self.rope_cache[rope_key] + else: + video_freq = self._compute_video_freqs(frame, height, width, idx) + video_freq = video_freq.to(device) + vid_freqs.append(video_freq) + + if self.scale_rope: + max_vid_index = max(height // 2, width // 2, max_vid_index) + else: + max_vid_index = max(height, width, max_vid_index) + + max_len = max(txt_seq_lens) + txt_freqs = self.pos_freqs[max_vid_index : max_vid_index + max_len, ...] + vid_freqs = torch.cat(vid_freqs, dim=0) + + return vid_freqs, txt_freqs + + @functools.cache + def _compute_video_freqs(self, frame, height, width, idx=0): + seq_lens = frame * height * width + freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1) + freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1) + + freqs_frame = freqs_pos[0][idx : idx + frame].view(frame, 1, 1, -1).expand(frame, height, width, -1) + if self.scale_rope: + freqs_height = torch.cat( + [freqs_neg[1][-(height - height // 2) :], freqs_pos[1][: height // 2]], + dim=0, + ) + freqs_height = freqs_height.view(1, height, 1, -1).expand(frame, height, width, -1) + freqs_width = torch.cat( + [freqs_neg[2][-(width - width // 2) :], freqs_pos[2][: width // 2]], + dim=0, + ) + freqs_width = freqs_width.view(1, 1, width, -1).expand(frame, height, width, -1) + else: + freqs_height = freqs_pos[1][:height].view(1, height, 1, -1).expand(frame, height, width, -1) + freqs_width = freqs_pos[2][:width].view(1, 1, width, -1).expand(frame, height, width, -1) + + freqs = torch.cat([freqs_frame, freqs_height, freqs_width], dim=-1).reshape(seq_lens, -1) + return freqs.clone().contiguous() + + +class QwenImageCrossAttention(nn.Module): + def __init__( + self, + dim: int, # query_dim + num_heads: int, + head_dim: int, + window_size=(-1, -1), + added_kv_proj_dim: int = None, + out_bias: bool = True, + qk_norm=True, # rmsnorm + eps=1e-6, + pre_only=False, + context_pre_only: bool = False, + parallel_attention=False, + out_dim: int = None, + ) -> None: + assert dim % num_heads == 0 + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.window_size = window_size + self.qk_norm = qk_norm + self.eps = eps + self.parallel_attention = parallel_attention + + # layers + # self.to_q = ReplicatedLinear(dim, dim) + # self.to_k = ReplicatedLinear(dim, dim) + # self.to_v = ReplicatedLinear(dim, dim) + self.to_qkv = QKVParallelLinear( + hidden_size=dim, + head_size=self.head_dim, + total_num_heads=num_heads, + disable_tp=True, + ) + self.norm_q = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity() + self.norm_k = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity() + self.inner_dim = out_dim if out_dim is not None else head_dim * num_heads + self.inner_kv_dim = self.inner_dim + if added_kv_proj_dim is not None: + assert context_pre_only is not None + # self.add_k_proj = ReplicatedLinear(added_kv_proj_dim, self.inner_kv_dim, bias=True) + # self.add_v_proj = ReplicatedLinear(added_kv_proj_dim, self.inner_kv_dim, bias=True) + # self.add_q_proj = ReplicatedLinear( + # added_kv_proj_dim, self.inner_dim, bias=True + # ) + self.add_kv_proj = QKVParallelLinear( + added_kv_proj_dim, + head_size=self.inner_kv_dim // self.num_heads, + total_num_heads=self.num_heads, + disable_tp=True, + ) + + if context_pre_only is not None and not context_pre_only: + self.to_add_out = ReplicatedLinear(self.inner_dim, self.dim, bias=out_bias) + else: + self.to_add_out = None + + if not pre_only: + self.to_out = nn.ModuleList([]) + self.to_out.append(ReplicatedLinear(self.inner_dim, self.dim, bias=out_bias)) + else: + self.to_out = None + + self.norm_added_q = RMSNorm(head_dim, eps=eps) + self.norm_added_k = RMSNorm(head_dim, eps=eps) + + # self.attn = Attention( + # num_heads=num_heads, + # head_size=self.head_dim, + # scale=1.0, # softmax_scale + # attn_type=AttentionType.ENCODER, + # ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + image_rotary_emb: tuple[torch.Tensor, torch.Tensor], + **cross_attention_kwargs, + ): + seq_len_txt = encoder_hidden_states.shape[1] + + # Compute QKV for image stream (sample projections) + qkv, _ = self.to_qkv(hidden_states) + img_query, img_key, img_value = qkv.chunk(3, dim=-1) + + # Compute QKV for text stream (context projections) + qkv, _ = self.add_kv_proj(encoder_hidden_states) + txt_query, txt_key, txt_value = qkv.chunk(3, dim=-1) + + # Reshape for multi-head attention + img_query = img_query.unflatten(-1, (self.num_heads, -1)) + img_key = img_key.unflatten(-1, (self.num_heads, -1)) + img_value = img_value.unflatten(-1, (self.num_heads, -1)) + + txt_query = txt_query.unflatten(-1, (self.num_heads, -1)) + txt_key = txt_key.unflatten(-1, (self.num_heads, -1)) + txt_value = txt_value.unflatten(-1, (self.num_heads, -1)) + + # Apply QK normalization + img_query = self.norm_q(img_query) + img_key = self.norm_k(img_key) + txt_query = self.norm_added_q(txt_query) + txt_key = self.norm_added_k(txt_key) + + # Apply RoPE + if image_rotary_emb is not None: + img_freqs, txt_freqs = image_rotary_emb + img_query = apply_rotary_emb_qwen(img_query, img_freqs, use_real=False) + img_key = apply_rotary_emb_qwen(img_key, img_freqs, use_real=False) + txt_query = apply_rotary_emb_qwen(txt_query, txt_freqs, use_real=False) + txt_key = apply_rotary_emb_qwen(txt_key, txt_freqs, use_real=False) + + # Concatenate for joint attention + # Order: [text, image] + joint_query = torch.cat([txt_query, img_query], dim=1) + joint_key = torch.cat([txt_key, img_key], dim=1) + joint_value = torch.cat([txt_value, img_value], dim=1) + + # Compute joint attention + # joint_hidden_states = self.attn( + # joint_query, + # joint_key, + # joint_value, + # ) + joint_hidden_states = dispatch_attention_fn( + joint_query, + joint_key, + joint_value, + dropout_p=0.0, + is_causal=False, + ) + + # Reshape back + joint_hidden_states = joint_hidden_states.flatten(2, 3) + joint_hidden_states = joint_hidden_states.to(joint_query.dtype) + + # Split attention outputs back + txt_attn_output = joint_hidden_states[:, :seq_len_txt, :] # Text part + img_attn_output = joint_hidden_states[:, seq_len_txt:, :] # Image part + + # Apply output projections + img_attn_output, _ = self.to_out[0](img_attn_output) + if len(self.to_out) > 1: + (img_attn_output,) = self.to_out[1](img_attn_output) # dropout + + txt_attn_output, _ = self.to_add_out(txt_attn_output) + + return img_attn_output, txt_attn_output + + +class QwenImageTransformerBlock(nn.Module): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + qk_norm: str = "rms_norm", + eps: float = 1e-6, + ): + super().__init__() + + self.dim = dim + self.num_attention_heads = num_attention_heads + self.attention_head_dim = attention_head_dim + + # Image processing modules + self.img_mod = nn.Sequential( + nn.SiLU(), + nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2 + ) + self.img_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + # self.attn = Attention( + # query_dim=dim, + # cross_attention_dim=None, # Enable cross attention for joint computation + # added_kv_proj_dim=dim, # Enable added KV projections for text stream + # dim_head=attention_head_dim, + # heads=num_attention_heads, + # out_dim=dim, + # context_pre_only=False, + # bias=True, + # processor=QwenDoubleStreamAttnProcessor2_0(), + # qk_norm=qk_norm, + # eps=eps, + # ) + self.attn = QwenImageCrossAttention( + dim=dim, + num_heads=num_attention_heads, + added_kv_proj_dim=dim, + context_pre_only=False, + head_dim=attention_head_dim, + ) + self.img_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.img_mlp = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") + + # Text processing modules + self.txt_mod = nn.Sequential( + nn.SiLU(), + nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2 + ) + self.txt_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + # Text doesn't need separate attention - it's handled by img_attn joint computation + self.txt_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.txt_mlp = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") + + def _modulate(self, x, mod_params): + """Apply modulation to input tensor""" + shift, scale, gate = mod_params.chunk(3, dim=-1) + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1), gate.unsqueeze(1) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + encoder_hidden_states_mask: torch.Tensor, + temb: torch.Tensor, + image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + # Get modulation parameters for both streams + img_mod_params = self.img_mod(temb) # [B, 6*dim] + txt_mod_params = self.txt_mod(temb) # [B, 6*dim] + + # Split modulation parameters for norm1 and norm2 + img_mod1, img_mod2 = img_mod_params.chunk(2, dim=-1) # Each [B, 3*dim] + txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1) # Each [B, 3*dim] + + # Process image stream - norm1 + modulation + img_normed = self.img_norm1(hidden_states) + img_modulated, img_gate1 = self._modulate(img_normed, img_mod1) + + # Process text stream - norm1 + modulation + txt_normed = self.txt_norm1(encoder_hidden_states) + txt_modulated, txt_gate1 = self._modulate(txt_normed, txt_mod1) + + # Use QwenAttnProcessor2_0 for joint attention computation + # This directly implements the DoubleStreamLayerMegatron logic: + # 1. Computes QKV for both streams + # 2. Applies QK normalization and RoPE + # 3. Concatenates and runs joint attention + # 4. Splits results back to separate streams + joint_attention_kwargs = joint_attention_kwargs or {} + attn_output = self.attn( + hidden_states=img_modulated, # Image stream (will be processed as "sample") + encoder_hidden_states=txt_modulated, # Text stream (will be processed as "context") + encoder_hidden_states_mask=encoder_hidden_states_mask, + image_rotary_emb=image_rotary_emb, + **joint_attention_kwargs, + ) + + # QwenAttnProcessor2_0 returns (img_output, txt_output) when encoder_hidden_states is provided + img_attn_output, txt_attn_output = attn_output + + # Apply attention gates and add residual (like in Megatron) + hidden_states = hidden_states + img_gate1 * img_attn_output + encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output + + # Process image stream - norm2 + MLP + img_normed2 = self.img_norm2(hidden_states) + img_modulated2, img_gate2 = self._modulate(img_normed2, img_mod2) + img_mlp_output = self.img_mlp(img_modulated2) + hidden_states = hidden_states + img_gate2 * img_mlp_output + + # Process text stream - norm2 + MLP + txt_normed2 = self.txt_norm2(encoder_hidden_states) + txt_modulated2, txt_gate2 = self._modulate(txt_normed2, txt_mod2) + txt_mlp_output = self.txt_mlp(txt_modulated2) + encoder_hidden_states = encoder_hidden_states + txt_gate2 * txt_mlp_output + + # Clip to prevent overflow for fp16 + if encoder_hidden_states.dtype == torch.float16: + encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504) + if hidden_states.dtype == torch.float16: + hidden_states = hidden_states.clip(-65504, 65504) + + return encoder_hidden_states, hidden_states + + +class QwenImageTransformer2DModel(nn.Module): + """ + The Transformer model introduced in Qwen. + + Args: + patch_size (`int`, defaults to `2`): + Patch size to turn the input data into small patches. + in_channels (`int`, defaults to `64`): + The number of channels in the input. + out_channels (`int`, *optional*, defaults to `None`): + The number of channels in the output. If not specified, it defaults to `in_channels`. + num_layers (`int`, defaults to `60`): + The number of layers of dual stream DiT blocks to use. + attention_head_dim (`int`, defaults to `128`): + The number of dimensions to use for each attention head. + num_attention_heads (`int`, defaults to `24`): + The number of attention heads to use. + joint_attention_dim (`int`, defaults to `3584`): + The number of dimensions to use for the joint attention (embedding/channel dimension of + `encoder_hidden_states`). + guidance_embeds (`bool`, defaults to `False`): + Whether to use guidance embeddings for guidance-distilled variant of the model. + axes_dims_rope (`tuple[int]`, defaults to `(16, 56, 56)`): + The dimensions to use for the rotary positional embeddings. + """ + + # _supports_gradient_checkpointing = True + # _no_split_modules = ["QwenImageTransformerBlock"] + # _skip_layerwise_casting_patterns = ["pos_embed", "norm"] + # _repeated_blocks = ["QwenImageTransformerBlock"] + def __init__( + self, + patch_size: int = 2, + in_channels: int = 64, + out_channels: Optional[int] = 16, + num_layers: int = 60, + attention_head_dim: int = 128, + num_attention_heads: int = 24, + joint_attention_dim: int = 3584, + guidance_embeds: bool = False, # TODO: this should probably be removed + axes_dims_rope: tuple[int, int, int] = (16, 56, 56), + ): + super().__init__() + self.config = None + self.in_channels = in_channels + self.out_channels = out_channels or in_channels + self.inner_dim = num_attention_heads * attention_head_dim + self.guidance_embeds = guidance_embeds + + self.pos_embed = QwenEmbedRope(theta=10000, axes_dim=list(axes_dims_rope), scale_rope=True) + + self.time_text_embed = QwenTimestepProjEmbeddings(embedding_dim=self.inner_dim) + + self.txt_norm = RMSNorm(joint_attention_dim, eps=1e-6) + + self.img_in = nn.Linear(in_channels, self.inner_dim) + self.txt_in = nn.Linear(joint_attention_dim, self.inner_dim) + + self.transformer_blocks = nn.ModuleList( + [ + QwenImageTransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + ) + for _ in range(num_layers) + ] + ) + + self.norm_out = AdaLayerNormContinuous(self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6) + self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + encoder_hidden_states_mask: torch.Tensor = None, + timestep: torch.LongTensor = None, + img_shapes: Optional[list[tuple[int, int, int]]] = None, + txt_seq_lens: Optional[list[int]] = None, + guidance: torch.Tensor = None, # TODO: this should probably be removed + attention_kwargs: Optional[dict[str, Any]] = None, + return_dict: bool = True, + ) -> Union[torch.Tensor, Transformer2DModelOutput]: + """ + The [`QwenTransformer2DModel`] forward method. + + Args: + hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`): + Input `hidden_states`. + encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`): + Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. + encoder_hidden_states_mask (`torch.Tensor` of shape `(batch_size, text_sequence_length)`): + Mask of the input conditions. + timestep ( `torch.LongTensor`): + Used to indicate denoising step. + attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain + tuple. + + Returns: + If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a + `tuple` where the first element is the sample tensor. + """ + # if attention_kwargs is not None: + # attention_kwargs = attention_kwargs.copy() + # lora_scale = attention_kwargs.pop("scale", 1.0) + # else: + # lora_scale = 1.0 + + hidden_states = self.img_in(hidden_states) + + # Ensure timestep tensor is on the same device and dtype as hidden_states + timestep = timestep.to(device=hidden_states.device, dtype=hidden_states.dtype) + encoder_hidden_states = self.txt_norm(encoder_hidden_states) + encoder_hidden_states = self.txt_in(encoder_hidden_states) + + if guidance is not None: + guidance = guidance.to(hidden_states.dtype) * 1000 + + temb = ( + self.time_text_embed(timestep, hidden_states) + if guidance is None + else self.time_text_embed(timestep, guidance, hidden_states) + ) + + image_rotary_emb = self.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device) + + for index_block, block in enumerate(self.transformer_blocks): + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + encoder_hidden_states_mask=encoder_hidden_states_mask, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=attention_kwargs, + ) + + # Use only the image part (hidden_states) from the dual-stream blocks + hidden_states = self.norm_out(hidden_states, temb) + output = self.proj_out(hidden_states) + + return Transformer2DModelOutput(sample=output) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + # Collect incoming weights into a dict for flexible mapping and grouping + weights_map: dict[str, torch.Tensor] = {name: tensor for name, tensor in weights} + + stacked_params_mapping = [ + ( + "to_qkv", + [ + {"aliases": ["q_proj", "to_q", "q", "query"], "order": 0}, + {"aliases": ["k_proj", "to_k", "k", "key"], "order": 1}, + {"aliases": ["v_proj", "to_v", "v", "value"], "order": 2}, + ], + ), + ( + "add_kv_proj", + [ + {"aliases": ["add_q_proj", "add_q"], "order": 0}, + {"aliases": ["add_k_proj", "add_k"], "order": 1}, + {"aliases": ["add_v_proj", "add_v"], "order": 2}, + ], + ), + ] + + # Helper that attempts to gather shards per base key using aliases and merges them + for target, shards in stacked_params_mapping: + base_candidates: dict[str, dict[str, dict[int, tuple[str, torch.Tensor]]]] = {} + for key in list(weights_map.keys()): + for shard in shards: + for alias in shard["aliases"]: + weight_suffix = f".{alias}.weight" + bias_suffix = f".{alias}.bias" + if key.endswith(weight_suffix): + base = key[: -len(weight_suffix)] + base_entry = base_candidates.setdefault(base, {"weight": {}, "bias": {}}) + base_entry["weight"][shard["order"]] = ( + key, + weights_map[key], + ) + break + if key.endswith(bias_suffix): + base = key[: -len(bias_suffix)] + base_entry = base_candidates.setdefault(base, {"weight": {}, "bias": {}}) + base_entry["bias"][shard["order"]] = (key, weights_map[key]) + break + + for base, entry in base_candidates.items(): + if len(entry["weight"]) == len(shards): + ordered = [entry["weight"][order] for order in sorted(entry["weight"])] + combined = torch.cat([tensor for _, tensor in ordered], dim=0) + for key, _ in ordered: + weights_map.pop(key, None) + weights_map[base + f".{target}.weight"] = combined + if len(entry["bias"]) == len(shards): + ordered_bias = [entry["bias"][order] for order in sorted(entry["bias"])] + bcombined = torch.cat([tensor for _, tensor in ordered_bias], dim=0) + for key, _ in ordered_bias: + weights_map.pop(key, None) + weights_map[base + f".{target}.bias"] = bcombined + + # Now perform the parameter copy using the possibly-updated weights_map + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights_map.items(): + # Handle potential name mismatches for FeedForward layers + if name.endswith(".proj.weight") and name not in params_dict: + mapped_name = name.replace(".proj.weight", ".weight") + elif name.endswith(".proj.bias") and name not in params_dict: + mapped_name = name.replace(".proj.bias", ".bias") + else: + mapped_name = name + + if mapped_name in params_dict: + param = params_dict[mapped_name] + with torch.no_grad(): + param.copy_(loaded_weight) + else: + logger.warning(f"Weight {name} (mapped to {mapped_name}) not found in model params.") diff --git a/vllm_omni/diffusion/omni_diffusion.py b/vllm_omni/diffusion/omni_diffusion.py new file mode 100644 index 000000000..c75d7da22 --- /dev/null +++ b/vllm_omni/diffusion/omni_diffusion.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import logging +from dataclasses import fields + +from vllm.logger import init_logger +from vllm.transformers_utils.config import get_hf_file_to_dict + +from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.diffusion.diffusion_engine import DiffusionEngine +from vllm_omni.diffusion.request import OmniDiffusionRequest + +# TODO configure logging properly +logging.basicConfig(level=logging.INFO) + +logger = init_logger(__name__) + + +def prepare_requests(prompt: str | list[str], **kwargs): + field_names = {f.name for f in fields(OmniDiffusionRequest)} + + init_kwargs = {"prompt": prompt} + + for key, value in kwargs.items(): + if key in field_names: + init_kwargs[key] = value + + return OmniDiffusionRequest(**init_kwargs) + + +class OmniDiffusion: + """ + It is the main class to interact with vLLM-Omni diffusion models. + It acts as a high-level interface that prepares requests and + delegates the actual diffusion process to the DiffusionEngine. + + You can pass either an `OmniDiffusionConfig` via `od_config`, or + pass kwargs such as `model="Qwen/Qwen-Image"`, + which will be forwarded to `OmniDiffusionConfig.from_kwargs`. + """ + + def __init__(self, od_config: OmniDiffusionConfig | None = None, **kwargs): + if od_config is None: + od_config = OmniDiffusionConfig.from_kwargs(**kwargs) + elif isinstance(od_config, dict): + od_config = OmniDiffusionConfig.from_kwargs(**od_config) + + self.od_config = od_config + + config_dict = get_hf_file_to_dict( + "model_index.json", + od_config.model, + ) + od_config.model_class_name = config_dict.get("_class_name", None) + + self.engine: DiffusionEngine = DiffusionEngine.make_engine(od_config) + + def generate( + self, + prompt: str | list[str], + **kwargs, + ): + prompts = [] + if isinstance(prompt, str): + prompts.append(prompt) + elif isinstance(prompt, list): + prompts.extend(prompt) + else: + raise ValueError("Prompt must be a string or a list of strings") + + requests: list[OmniDiffusionRequest] = [] + for p in prompts: + requests.append( + prepare_requests( + p, + **kwargs, + ) + ) + logger.info(f"Prepared {len(requests)} requests for generation.") + return self._run_engine(requests) + + def _run_engine(self, requests: list[OmniDiffusionRequest]): + return self.engine.step(requests) diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py new file mode 100644 index 000000000..0d834b2a0 --- /dev/null +++ b/vllm_omni/diffusion/registry.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import importlib + +from vllm_omni.diffusion.data import OmniDiffusionConfig + +_DIFFUSION_MODELS = { + # arch:(mod_folder, mod_relname, cls_name) + "QwenImagePipeline": ( + "qwen_image", + "pipeline_qwen_image", + "QwenImagePipeline", + ), +} + + +def initialize_model( + od_config: OmniDiffusionConfig, +): + if od_config.model_class_name in _DIFFUSION_MODELS: + mod_folder, mod_relname, cls_name = _DIFFUSION_MODELS[od_config.model_class_name] + module_name = f"vllm_omni.diffusion.models.{mod_folder}.{mod_relname}" + module = importlib.import_module(module_name) + model_class = getattr(module, cls_name) + model = model_class(od_config=od_config, prefix=mod_relname) + return model + else: + raise ValueError(f"Model class {od_config.model_class_name} not found in diffusion model registry.") + + +_DIFFUSION_POST_PROCESS_FUNCS = { + # arch: post_process_func + # `post_process_func` function must be placed in {mod_folder}/{mod_relname}.py, + # where mod_folder and mod_relname are defined and mapped using `_DIFFUSION_MODELS` via the `arch` key + "QwenImagePipeline": "get_qwen_image_post_process_func", +} + + +def get_diffusion_post_process_func(od_config: OmniDiffusionConfig): + if od_config.model_class_name in _DIFFUSION_POST_PROCESS_FUNCS: + mod_folder, mod_relname, _ = _DIFFUSION_MODELS[od_config.model_class_name] + func_name = _DIFFUSION_POST_PROCESS_FUNCS[od_config.model_class_name] + module_name = f"vllm_omni.diffusion.models.{mod_folder}.{mod_relname}" + module = importlib.import_module(module_name) + post_process_func = getattr(module, func_name) + return post_process_func(od_config) + else: + raise ValueError( + f"Post process function for model class {od_config.model_class_name} not found in diffusion model registry." + ) diff --git a/vllm_omni/diffusion/request.py b/vllm_omni/diffusion/request.py new file mode 100644 index 000000000..100f8a50c --- /dev/null +++ b/vllm_omni/diffusion/request.py @@ -0,0 +1,179 @@ +# adapted from sglang and fastvideo +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pprint +from dataclasses import asdict, dataclass, field +from typing import Any + +import PIL.Image +import torch + + +@dataclass +class OmniDiffusionRequest: + """ + Complete state passed through the pipeline execution. + + This dataclass contains all information needed during the diffusion pipeline + execution, allowing methods to update specific components without needing + to manage numerous individual parameters. + """ + + # TODO(will): double check that args are separate from server_args + # properly. Also maybe think about providing an abstraction for pipeline + # specific arguments. + # data_type: DataType + + request_id: str | None = None + + generator: torch.Generator | list[torch.Generator] | None = None + + # Image inputs + image_path: str | None = None + # Image encoder hidden states + image_embeds: list[torch.Tensor] = field(default_factory=list) + pil_image: torch.Tensor | PIL.Image.Image | None = None + pixel_values: torch.Tensor | PIL.Image.Image | None = None + preprocessed_image: torch.Tensor | None = None + + # Text inputs + prompt: str | list[str] | None = None + negative_prompt: str | list[str] | None = None + prompt_path: str | None = None + output_path: str = "outputs/" + # without extension + output_file_name: str | None = None + output_file_ext: str | None = None + # Primary encoder embeddings + prompt_embeds: list[torch.Tensor] | torch.Tensor = field(default_factory=list) + negative_prompt_embeds: list[torch.Tensor] | None = None + prompt_attention_mask: list[torch.Tensor] | None = None + negative_attention_mask: list[torch.Tensor] | None = None + clip_embedding_pos: list[torch.Tensor] | None = None + clip_embedding_neg: list[torch.Tensor] | None = None + + pooled_embeds: list[torch.Tensor] = field(default_factory=list) + neg_pooled_embeds: list[torch.Tensor] = field(default_factory=list) + + # Additional text-related parameters + max_sequence_length: int | None = None + prompt_template: dict[str, Any] | None = None + do_classifier_free_guidance: bool = False + + # Batch info + num_outputs_per_prompt: int = 1 + seed: int | None = None + seeds: list[int] | None = None + + # Tracking if embeddings are already processed + is_prompt_processed: bool = False + + # Latent tensors + latents: torch.Tensor | None = None + raw_latent_shape: torch.Tensor | None = None + noise_pred: torch.Tensor | None = None + image_latent: torch.Tensor | None = None + + # Latent dimensions + height_latents: list[int] | int | None = None + width_latents: list[int] | int | None = None + num_frames: list[int] | int = 1 # Default for image models + num_frames_round_down: bool = False # Whether to round down num_frames if it's not divisible by num_gpus + + # Original dimensions (before VAE scaling) + height: list[int] | int | None = None + width: list[int] | int | None = None + fps: list[int] | int | None = None + height_not_provided: bool = False + width_not_provided: bool = False + + # Timesteps + timesteps: torch.Tensor | None = None + timestep: torch.Tensor | float | int | None = None + step_index: int | None = None + boundary_ratio: float | None = None + + # Scheduler parameters + num_inference_steps: int = 50 + guidance_scale: float = 1.0 + guidance_scale_2: float | None = None + guidance_rescale: float = 0.0 + eta: float = 0.0 + sigmas: list[float] | None = None + + true_cfg_scale: float | None = None # qwen-image specific now + + n_tokens: int | None = None + + # Other parameters that may be needed by specific schedulers + extra_step_kwargs: dict[str, Any] = field(default_factory=dict) + + # Component modules (populated by the pipeline) + modules: dict[str, Any] = field(default_factory=dict) + + return_trajectory_latents: bool = False + return_trajectory_decoded: bool = False + trajectory_timesteps: list[torch.Tensor] | None = None + trajectory_latents: torch.Tensor | None = None + + # Extra parameters that might be needed by specific pipeline implementations + extra: dict[str, Any] = field(default_factory=dict) + + # Misc + save_output: bool = True + return_frames: bool = False + + # TeaCache parameters + enable_teacache: bool = False + # teacache_params: TeaCacheParams | WanTeaCacheParams | None = None + + # STA parameters + STA_param: list | None = None + is_cfg_negative: bool = False + mask_search_final_result_pos: list[list] | None = None + mask_search_final_result_neg: list[list] | None = None + + # VSA parameters + VSA_sparsity: float = 0.0 + # perf_logger: PerformanceLogger | None = None + + # stage logging + # logging_info: PipelineLoggingInfo = field(default_factory=PipelineLoggingInfo) + + # profile + profile: bool = False + num_profiled_timesteps: int = 8 + + # debugging + debug: bool = False + + # results + output: torch.Tensor | None = None + + @property + def batch_size(self): + # Determine batch size + if isinstance(self.prompt, list): + batch_size = len(self.prompt) + elif self.prompt is not None: + batch_size = 1 + else: + batch_size = self.prompt_embeds[0].shape[0] + + # Adjust batch size for number of videos per prompt + batch_size *= self.num_outputs_per_prompt + return batch_size + + def __post_init__(self): + """Initialize dependent fields after dataclass initialization.""" + # Set do_classifier_free_guidance based on guidance scale and negative prompt + if self.guidance_scale > 1.0 and self.negative_prompt is not None: + self.do_classifier_free_guidance = True + if self.negative_prompt_embeds is None: + self.negative_prompt_embeds = [] + if self.guidance_scale_2 is None: + self.guidance_scale_2 = self.guidance_scale + + def __str__(self): + return pprint.pformat(asdict(self), indent=2, width=120) diff --git a/vllm_omni/diffusion/utils/__init__.py b/vllm_omni/diffusion/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_omni/diffusion/utils/hf_utils.py b/vllm_omni/diffusion/utils/hf_utils.py new file mode 100644 index 000000000..61c7f1982 --- /dev/null +++ b/vllm_omni/diffusion/utils/hf_utils.py @@ -0,0 +1,20 @@ +from functools import lru_cache + +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +def load_diffusers_config(model_name) -> dict: + config = DiffusionPipeline.load_config(model_name) + return config + + +@lru_cache +def is_diffusion_model(model_name: str) -> bool: + try: + load_diffusers_config(model_name) + return True + except Exception: + return False diff --git a/vllm_omni/diffusion/utils/network_utils.py b/vllm_omni/diffusion/utils/network_utils.py new file mode 100644 index 000000000..c3085e8f9 --- /dev/null +++ b/vllm_omni/diffusion/utils/network_utils.py @@ -0,0 +1,19 @@ +import socket + +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +def is_port_available(port): + """Return whether a port is available.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("", port)) + s.listen(1) + return True + except OSError: + return False + except OverflowError: + return False diff --git a/vllm_omni/diffusion/worker/gpu_worker.py b/vllm_omni/diffusion/worker/gpu_worker.py new file mode 100644 index 000000000..84ede7ffd --- /dev/null +++ b/vllm_omni/diffusion/worker/gpu_worker.py @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import multiprocessing as mp +import os + +import torch +import zmq +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.distributed.device_communicators.shm_broadcast import MessageQueue +from vllm.distributed.parallel_state import ( + init_distributed_environment, + initialize_model_parallel, +) +from vllm.logger import init_logger + +from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.registry import initialize_model +from vllm_omni.diffusion.request import OmniDiffusionRequest + +logger = init_logger(__name__) + + +class GPUWorker: + """ + A worker that executes the model on a single GPU. + """ + + def __init__( + self, + local_rank: int, + rank: int, + od_config: OmniDiffusionConfig, + ): + self.local_rank = local_rank + self.rank = rank + self.od_config = od_config + self.pipeline = None + + self.init_device_and_model() + + def init_device_and_model(self) -> None: + """Initialize the device and load the model.""" + world_size = self.od_config.num_gpus + rank = self.rank + # Set environment variables for distributed initialization + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(self.od_config.master_port) + os.environ["LOCAL_RANK"] = str(self.local_rank) + os.environ["RANK"] = str(rank) + os.environ["WORLD_SIZE"] = str(world_size) + + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + + # hack + vllm_config = VllmConfig() + vllm_config.parallel_config.tensor_parallel_size = self.od_config.num_gpus + set_current_vllm_config(vllm_config) + + init_distributed_environment(world_size=world_size, rank=rank) + initialize_model_parallel(tensor_model_parallel_size=world_size) + + with device: + self.pipeline = initialize_model(self.od_config) + self.pipeline.load_weights() + self.pipeline.eval() + logger.info(f"Worker {self.rank}: Initialized device, model, and distributed environment.") + logger.info(f"Worker {self.rank}: Model loaded successfully.") + + @torch.inference_mode() + def execute_model(self, reqs: list[OmniDiffusionRequest], od_config: OmniDiffusionConfig) -> DiffusionOutput: + """ + Execute a forward pass. + """ + assert self.pipeline is not None + # TODO: dealing with first req for now + req = reqs[0] + output = self.pipeline.forward(req) + return output + + +class WorkerProc: + """Wrapper that runs one Worker in a separate process.""" + + def __init__( + self, + od_config: OmniDiffusionConfig, + gpu_id: int, + broadcast_handle, + ): + self.od_config = od_config + + # Inter-process Communication + self.context = zmq.Context(io_threads=2) + + # Initialize MessageQueue reader from handle + self.mq = MessageQueue.create_from_handle(broadcast_handle, gpu_id) + + self.result_mq = None + self.result_mq_handle = None + + # Setup result sender (only for rank 0 for now, or whoever needs to reply) + # Assuming only rank 0 replies to scheduler as per original logic + if gpu_id == 0: + # Create MessageQueue for results (1 writer -> 1 reader) + # We assume the reader (SyncScheduler) will act as rank 0 + self.result_mq = MessageQueue(n_reader=1, n_local_reader=1, local_reader_ranks=[0]) + self.result_mq_handle = self.result_mq.export_handle() + logger.info(f"Worker {gpu_id} created result MessageQueue") + + assert od_config.master_port is not None + worker = GPUWorker( + local_rank=gpu_id, + rank=gpu_id, + od_config=od_config, + ) + self.worker = worker + self.gpu_id = gpu_id + self._running = True + + def return_result(self, output: DiffusionOutput): + """ + replies to client, only on rank 0 + """ + if self.result_mq is not None: + self.result_mq.enqueue(output) + + def recv_reqs(self): + """ + Receive requests from broadcast queue + """ + return self.mq.dequeue() + + # TODO: queueing, cancellation + def worker_busy_loop(self) -> None: + """Main busy loop for Multiprocessing Workers""" + + logger.info(f"Worker {self.gpu_id} ready to receive requests via shared memory") + + while self._running: + reqs = None + # 1: receive requests + try: + reqs = self.recv_reqs() + except Exception as e: + logger.error( + f"Error receiving requests in scheduler event loop: {e}", + exc_info=True, + ) + continue + + # 2: execute, make sure a reply is always sent + try: + output = self.worker.execute_model(reqs, self.od_config) + except Exception as e: + logger.error( + f"Error executing forward in event loop: {e}", + exc_info=True, + ) + output = DiffusionOutput(error=str(e)) + + try: + self.return_result(output) + except zmq.ZMQError as e: + # Reply failed; log and keep loop alive to accept future requests + logger.error(f"ZMQ error sending reply: {e}") + continue + + logger.info("event loop terminated.") + # if self.result_sender is not None: + # self.result_sender.close() + self.context.term() + + @staticmethod + def worker_main( + rank: int, + od_config: OmniDiffusionConfig, + pipe_writer: mp.connection.Connection, + broadcast_handle, + ) -> None: + """Worker initialization and execution loops.""" + + worker_proc = WorkerProc( + od_config, + gpu_id=rank, + broadcast_handle=broadcast_handle, + ) + logger.info(f"Worker {rank}: Scheduler loop started.") + pipe_writer.send( + { + "status": "ready", + "result_handle": worker_proc.result_mq_handle if rank == 0 else None, + } + ) + worker_proc.worker_busy_loop() + logger.info(f"Worker {rank}: Shutdown complete.") diff --git a/vllm_omni/entrypoints/omni.py b/vllm_omni/entrypoints/omni.py new file mode 100644 index 000000000..f87c89259 --- /dev/null +++ b/vllm_omni/entrypoints/omni.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# unify OmniLLM and OmniDiffusion entrypoints + +from vllm_omni.diffusion.omni_diffusion import OmniDiffusion +from vllm_omni.diffusion.utils.hf_utils import is_diffusion_model +from vllm_omni.entrypoints.omni_llm import OmniLLM + + +class Omni: + def __init__(self, *args, **kwargs): + model = args[0] if args else kwargs.get("model", "") + if is_diffusion_model(model): + self.instance: OmniLLM | OmniDiffusion = OmniDiffusion(*args, **kwargs) + else: + self.instance: OmniLLM | OmniDiffusion = OmniLLM(*args, **kwargs) + + def __getattr__(self, name): + """Delegate attribute access to the chosen backend instance.""" + return getattr(self.instance, name) + + def generate(self, *args, **kwargs): + """Convenience wrapper to call `generate` on the backend if available.""" + if hasattr(self.instance, "generate"): + return getattr(self.instance, "generate")(*args, **kwargs) + raise AttributeError(f"'{self.instance.__class__.__name__}' has no attribute 'generate'") diff --git a/vllm_omni/entrypoints/stage_utils.py b/vllm_omni/entrypoints/stage_utils.py index 36ad2b0ab..73179259a 100644 --- a/vllm_omni/entrypoints/stage_utils.py +++ b/vllm_omni/entrypoints/stage_utils.py @@ -84,7 +84,7 @@ def set_stage_gpu_devices(stage_id: int, devices: str | int | None) -> None: ) try: - import torch # noqa: WPS433 + import torch if torch.cuda.is_available(): try: