[Diffusion] Qwen image support (#82)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>

docs/contributing/DOCS_GUIDE.md

@@ -38,7 +38,7 @@ The documentation automatically extracts docstrings from the code using mkdocstr
 Example docstring:
 
 ```python
-class OmniLLM:
+class Omni:
     """Main entry point for vLLM-omni inference.
 
     This class provides a high-level interface for running multi-modal
@@ -50,7 +50,7 @@ class OmniLLM:
         **kwargs: Additional arguments passed to the engine
 
     Example:
-        >>> llm = OmniLLM(model="Qwen/Qwen2.5-Omni")
+        >>> llm = Omni(model="Qwen/Qwen2.5-Omni")
         >>> outputs = llm.generate(prompts="Hello")
     """
 ```

examples/offline_inference/diffusion/qwen_image.py

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import torch
+
+from vllm_omni.entrypoints.omni import Omni
+
+model_name = "Qwen/Qwen-Image"
+prompt = "a cup of coffee on the table"
+if __name__ == "__main__":
+    m = Omni(
+        model=model_name,
+    )
+    image = m.generate(
+        prompt,
+        height=1024,
+        width=1024,
+        generator=torch.Generator(device="cuda").manual_seed(42),
+    )
+    image[0].save("qwen_image_output.png")

examples/offline_inference/qwen2_5_omni/end2end.py

@@ -16,7 +16,7 @@ from vllm.multimodal.image import convert_image_mode
 from vllm.sampling_params import SamplingParams
 from vllm.utils import FlexibleArgumentParser
 
-from vllm_omni import OmniLLM
+from vllm_omni import Omni
 
 SEED = 42
 
@@ -140,7 +140,7 @@ def main(args):
     model_name = "Qwen/Qwen2.5-Omni-7B"
     query_result = query_map[args.query_type]()
 
-    omni_llm = OmniLLM(
+    omni_llm = Omni(
         model=model_name,
         log_stats=args.enable_stats,
         log_file=("omni_llm_pipeline.log" if args.enable_stats else None),

pyproject.toml

@@ -33,6 +33,8 @@ dependencies = [
     # "decord>=0.6.0", # install manually
     "librosa>=0.11.0",
     "resampy>=0.4.3",
+    "diffusers==0.35.2",
+    "accelerate==1.12.0",
     # "vllm==0.11.0",  # TODO: fix the entrypoints overwrite problem
 ]
 
@@ -160,5 +162,7 @@ extend-ignore-identifiers-re = [
     ".*_thw",
     ".*thw",
     "ein",
-    ".*arange"
+    ".*arange",
+    ".*MoBA",
+    ".*temperal_downsample"
 ]

vllm_omni/__init__.py

@@ -23,7 +23,7 @@ from .config import OmniModelConfig
 from .entrypoints.async_omni_llm import AsyncOmniLLM
 
 # Main entry points
-from .entrypoints.omni_llm import OmniLLM
+from .entrypoints.omni import Omni
 
 from .version import __version__, __version_tuple__  # isort:skip
 
@@ -32,7 +32,7 @@ __all__ = [
     "__version__",
     "__version_tuple__",
     # Main components
-    "OmniLLM",
+    "Omni",
     "AsyncOmniLLM",
     # Configuration
     "OmniModelConfig",

vllm_omni/diffusion/data.py

@@ -0,0 +1,210 @@
+# adapted from sglang and fastvideo
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import enum
+import random
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+import torch
+from vllm.logger import init_logger
+
+from vllm_omni.diffusion.utils.network_utils import is_port_available
+
+logger = init_logger(__name__)
+
+
+@dataclass
+class OmniDiffusionConfig:
+    # Model and path configuration (for convenience)
+    model: str
+
+    model_class_name: str | None = None
+
+    # Attention
+    # attention_backend: str = None
+
+    # Running mode
+    # mode: ExecutionMode = ExecutionMode.INFERENCE
+
+    # Workload type
+    # workload_type: WorkloadType = WorkloadType.T2V
+
+    # Cache strategy
+    cache_strategy: str = "none"
+
+    # Distributed executor backend
+    distributed_executor_backend: str = "mp"
+    nccl_port: int | None = None
+
+    # HuggingFace specific parameters
+    trust_remote_code: bool = False
+    revision: str | None = None
+
+    # Parallelism
+    num_gpus: int = 1
+    tp_size: int = -1
+    sp_degree: int = -1
+    # sequence parallelism
+    ulysses_degree: int | None = None
+    ring_degree: int | None = None
+    # data parallelism
+    # number of data parallelism groups
+    dp_size: int = 1
+    # number of gpu in a dp group
+    dp_degree: int = 1
+    # cfg parallel
+    enable_cfg_parallel: bool = False
+
+    hsdp_replicate_dim: int = 1
+    hsdp_shard_dim: int = -1
+    dist_timeout: int | None = None  # timeout for torch.distributed
+
+    # pipeline_config: PipelineConfig = field(default_factory=PipelineConfig, repr=False)
+
+    # LoRA parameters
+    # (Wenxuan) prefer to keep it here instead of in pipeline config to not make it complicated.
+    lora_path: str | None = None
+    lora_nickname: str = "default"  # for swapping adapters in the pipeline
+    # can restrict layers to adapt, e.g. ["q_proj"]
+    # Will adapt only q, k, v, o by default.
+    lora_target_modules: list[str] | None = None
+
+    output_type: str = "pil"
+
+    # CPU offload parameters
+    dit_cpu_offload: bool = True
+    use_fsdp_inference: bool = False
+    text_encoder_cpu_offload: bool = True
+    image_encoder_cpu_offload: bool = True
+    vae_cpu_offload: bool = True
+    pin_cpu_memory: bool = True
+
+    # STA (Sliding Tile Attention) parameters
+    mask_strategy_file_path: str | None = None
+    # STA_mode: STA_Mode = STA_Mode.STA_INFERENCE
+    skip_time_steps: int = 15
+
+    # Compilation
+    enable_torch_compile: bool = False
+
+    disable_autocast: bool = False
+
+    # VSA parameters
+    VSA_sparsity: float = 0.0  # inference/validation sparsity
+
+    # V-MoBA parameters
+    moba_config_path: str | None = None
+    # moba_config: dict[str, Any] = field(default_factory=dict)
+
+    # Master port for distributed inference
+    # TODO: do not hard code
+    master_port: int | None = None
+
+    # http server endpoint config, would be ignored in local mode
+    host: str | None = None
+    port: int | None = None
+
+    scheduler_port: int = 5555
+
+    # Stage verification
+    enable_stage_verification: bool = True
+
+    # Prompt text file for batch processing
+    prompt_file_path: str | None = None
+
+    # model paths for correct deallocation
+    model_paths: dict[str, str] = field(default_factory=dict)
+    model_loaded: dict[str, bool] = field(
+        default_factory=lambda: {
+            "transformer": True,
+            "vae": True,
+        }
+    )
+    override_transformer_cls_name: str | None = None
+
+    # # DMD parameters
+    # dmd_denoising_steps: List[int] | None = field(default=None)
+
+    # MoE parameters used by Wan2.2
+    boundary_ratio: float | None = None
+
+    # Logging
+    log_level: str = "info"
+
+    def settle_port(self, port: int, port_inc: int = 42, max_attempts: int = 100) -> int:
+        """
+        Find an available port with retry logic.
+
+        Args:
+            port: Initial port to check
+            port_inc: Port increment for each attempt
+            max_attempts: Maximum number of attempts to find an available port
+
+        Returns:
+            An available port number
+
+        Raises:
+            RuntimeError: If no available port is found after max_attempts
+        """
+        attempts = 0
+        original_port = port
+
+        while attempts < max_attempts:
+            if is_port_available(port):
+                if attempts > 0:
+                    logger.info(f"Port {original_port} was unavailable, using port {port} instead")
+                return port
+
+            attempts += 1
+            if port < 60000:
+                port += port_inc
+            else:
+                # Wrap around with randomization to avoid collision
+                port = 5000 + random.randint(0, 1000)
+
+        raise RuntimeError(
+            f"Failed to find available port after {max_attempts} attempts (started from port {original_port})"
+        )
+
+    def __post_init__(self):
+        # TODO: remove hard code
+        initial_master_port = (self.master_port or 30005) + random.randint(0, 100)
+        self.master_port = self.settle_port(initial_master_port, 37)
+
+    @classmethod
+    def from_kwargs(cls, **kwargs: Any) -> "OmniDiffusionConfig":
+        return cls(**kwargs)
+
+
+@dataclass
+class DiffusionOutput:
+    """
+    Final output (after pipeline completion)
+    """
+
+    output: torch.Tensor | None = None
+    trajectory_timesteps: list[torch.Tensor] | None = None
+    trajectory_latents: torch.Tensor | None = None
+    trajectory_decoded: list[torch.Tensor] | None = None
+    error: str | None = None
+
+    post_process_func: Callable[..., Any] | None = None
+
+    # logged timings info, directly from Req.timings
+    # timings: Optional["RequestTimings"] = None
+
+
+class AttentionBackendEnum(enum.Enum):
+    FA = enum.auto()
+    SLIDING_TILE_ATTN = enum.auto()
+    TORCH_SDPA = enum.auto()
+    SAGE_ATTN = enum.auto()
+    SAGE_ATTN_THREE = enum.auto()
+    VIDEO_SPARSE_ATTN = enum.auto()
+    VMOBA_ATTN = enum.auto()
+    AITER = enum.auto()
+    NO_ATTENTION = enum.auto()
+
+    def __str__(self):
+        return self.name.lower()

vllm_omni/diffusion/diffusion_engine.py

@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import multiprocessing as mp
+
+from vllm.logger import init_logger
+
+from vllm_omni.diffusion.data import OmniDiffusionConfig
+from vllm_omni.diffusion.registry import get_diffusion_post_process_func
+from vllm_omni.diffusion.request import OmniDiffusionRequest
+from vllm_omni.diffusion.scheduler import scheduler
+from vllm_omni.diffusion.worker.gpu_worker import WorkerProc
+
+logger = init_logger(__name__)
+
+
+class DiffusionEngine:
+    """The diffusion engine for vLLM-Omni diffusion models."""
+
+    def __init__(self, od_config: OmniDiffusionConfig):
+        """Initialize the diffusion engine.
+
+        Args:
+            config: The configuration for the diffusion engine.
+        """
+        self.od_config = od_config
+
+        self.post_process_func = get_diffusion_post_process_func(od_config)
+
+        self._make_client()
+
+    def step(self, requests: list[OmniDiffusionRequest]):
+        try:
+            output = self.add_req_and_wait_for_response(requests)
+            if output.error:
+                raise Exception(f"{output.error}")
+            logger.info("Generation completed successfully.")
+            return self.post_process_func(output.output)
+        except Exception as e:
+            logger.error(f"Generation failed: {e}")
+            return None
+
+    @staticmethod
+    def make_engine(config: OmniDiffusionConfig) -> "DiffusionEngine":
+        """Factory method to create a DiffusionEngine instance.
+
+        Args:
+            config: The configuration for the diffusion engine.
+
+        Returns:
+            An instance of DiffusionEngine.
+        """
+        return DiffusionEngine(config)
+
+    def _make_client(self):
+        # TODO rename it
+        scheduler.initialize(self.od_config)
+
+        # Get the broadcast handle from the initialized scheduler
+        broadcast_handle = scheduler.get_broadcast_handle()
+
+        processes, result_handle = self._launch_workers(
+            broadcast_handle=broadcast_handle,
+        )
+
+        if result_handle is not None:
+            scheduler.initialize_result_queue(result_handle)
+        else:
+            logger.error("Failed to get result queue handle from workers")
+
+    def _launch_workers(self, broadcast_handle):
+        od_config = self.od_config
+        logger.info("Starting server...")
+
+        num_gpus = od_config.num_gpus
+        mp.set_start_method("spawn", force=True)
+        processes = []
+
+        # Launch all worker processes
+        scheduler_pipe_readers = []
+        scheduler_pipe_writers = []
+
+        for i in range(num_gpus):
+            reader, writer = mp.Pipe(duplex=False)
+            scheduler_pipe_writers.append(writer)
+            process = mp.Process(
+                target=WorkerProc.worker_main,
+                args=(
+                    i,  # rank
+                    od_config,
+                    writer,
+                    broadcast_handle,
+                ),
+                name=f"DiffusionWorker-{i}",
+                daemon=True,
+            )
+            scheduler_pipe_readers.append(reader)
+            process.start()
+            processes.append(process)
+
+        # Wait for all workers to be ready
+        scheduler_infos = []
+        result_handle = None
+        for writer in scheduler_pipe_writers:
+            writer.close()
+
+        for i, reader in enumerate(scheduler_pipe_readers):
+            try:
+                data = reader.recv()
+            except EOFError:
+                logger.error(f"Rank {i} scheduler is dead. Please check if there are relevant logs.")
+                processes[i].join()
+                logger.error(f"Exit code: {processes[i].exitcode}")
+                raise
+
+            if data["status"] != "ready":
+                raise RuntimeError("Initialization failed. Please see the error messages above.")
+
+            if i == 0:
+                result_handle = data.get("result_handle")
+
+            scheduler_infos.append(data)
+            reader.close()
+
+        logger.debug("All workers are ready")
+
+        return processes, result_handle
+
+    def add_req_and_wait_for_response(self, requests: list[OmniDiffusionRequest]):
+        return scheduler.add_req(requests)

vllm_omni/diffusion/distributed/utils.py

@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import os
+
+import torch
+
+
+def get_local_device() -> torch.device:
+    """Return the torch device for the current rank."""
+    return torch.device(f"cuda:{os.environ.get('LOCAL_RANK', 0)}")

vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py

@@ -0,0 +1,776 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import inspect
+import json
+import logging
+import math
+import os
+from typing import Any, Optional, Union
+
+import numpy as np
+import torch
+from diffusers.image_processor import VaeImageProcessor
+from diffusers.models.autoencoders.autoencoder_kl_qwenimage import (
+    AutoencoderKLQwenImage,
+)
+from diffusers.schedulers.scheduling_flow_match_euler_discrete import (
+    FlowMatchEulerDiscreteScheduler,
+)
+from diffusers.utils.torch_utils import randn_tensor
+from torch import nn
+from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer
+
+from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
+from vllm_omni.diffusion.distributed.utils import get_local_device
+from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
+    QwenImageTransformer2DModel,
+)
+from vllm_omni.diffusion.request import OmniDiffusionRequest
+from vllm_omni.model_executor.model_loader.weight_utils import (
+    download_weights_from_hf_specific,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def get_qwen_image_post_process_func(
+    od_config: OmniDiffusionConfig,
+):
+    model_name = od_config.model
+    model_path = download_weights_from_hf_specific(model_name, None, ["*"])
+    vae_config_path = os.path.join(model_path, "vae/config.json")
+    with open(vae_config_path) as f:
+        vae_config = json.load(f)
+        vae_scale_factor = 2 ** len(vae_config["temporal_downsample"]) if "temporal_downsample" in vae_config else 8
+
+    image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2)
+
+    def post_process_func(
+        images: torch.Tensor,
+    ):
+        return image_processor.postprocess(images)
+
+    return post_process_func
+
+
+def calculate_shift(
+    image_seq_len,
+    base_seq_len: int = 256,
+    max_seq_len: int = 4096,
+    base_shift: float = 0.5,
+    max_shift: float = 1.15,
+):
+    m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
+    b = base_shift - m * base_seq_len
+    mu = image_seq_len * m + b
+    return mu
+
+
+def retrieve_timesteps(
+    scheduler,
+    num_inference_steps: Optional[int] = None,
+    device: Optional[Union[str, torch.device]] = None,
+    timesteps: Optional[list[int]] = None,
+    sigmas: Optional[list[float]] = None,
+    **kwargs,
+) -> tuple[torch.Tensor, int]:
+    r"""
+    Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
+    custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
+
+    Args:
+        scheduler (`SchedulerMixin`):
+            The scheduler to get timesteps from.
+        num_inference_steps (`int`):
+            The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
+            must be `None`.
+        device (`str` or `torch.device`, *optional*):
+            The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
+        timesteps (`list[int]`, *optional*):
+            Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
+            `num_inference_steps` and `sigmas` must be `None`.
+        sigmas (`list[float]`, *optional*):
+            Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
+            `num_inference_steps` and `timesteps` must be `None`.
+
+    Returns:
+        `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
+        second element is the number of inference steps.
+    """
+    if timesteps is not None and sigmas is not None:
+        raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
+    if timesteps is not None:
+        accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
+        if not accepts_timesteps:
+            raise ValueError(
+                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
+                f" timestep schedules. Please check whether you are using the correct scheduler."
+            )
+        scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
+        timesteps = scheduler.timesteps
+        num_inference_steps = len(timesteps)
+    elif sigmas is not None:
+        accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
+        if not accept_sigmas:
+            raise ValueError(
+                f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
+                f" sigmas schedules. Please check whether you are using the correct scheduler."
+            )
+        scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
+        timesteps = scheduler.timesteps
+        num_inference_steps = len(timesteps)
+    else:
+        scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
+        timesteps = scheduler.timesteps
+    return timesteps, num_inference_steps
+
+
+def get_timestep_embedding(
+    timesteps: torch.Tensor,
+    embedding_dim: int,
+    flip_sin_to_cos: bool = False,
+    downscale_freq_shift: float = 1,
+    scale: float = 1,
+    max_period: int = 10000,
+) -> torch.Tensor:
+    """
+    This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings.
+
+    Args
+        timesteps (torch.Tensor):
+            a 1-D Tensor of N indices, one per batch element. These may be fractional.
+        embedding_dim (int):
+            the dimension of the output.
+        flip_sin_to_cos (bool):
+            Whether the embedding order should be `cos, sin` (if True) or `sin, cos` (if False)
+        downscale_freq_shift (float):
+            Controls the delta between frequencies between dimensions
+        scale (float):
+            Scaling factor applied to the embeddings.
+        max_period (int):
+            Controls the maximum frequency of the embeddings
+    Returns
+        torch.Tensor: an [N x dim] Tensor of positional embeddings.
+    """
+    assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array"
+
+    half_dim = embedding_dim // 2
+    exponent = -math.log(max_period) * torch.arange(start=0, end=half_dim, dtype=torch.float32, device=timesteps.device)
+    exponent = exponent / (half_dim - downscale_freq_shift)
+
+    emb = torch.exp(exponent).to(timesteps.dtype)
+    emb = timesteps[:, None].float() * emb[None, :]
+
+    # scale embeddings
+    emb = scale * emb
+
+    # concat sine and cosine embeddings
+    emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1)
+
+    # flip sine and cosine embeddings
+    if flip_sin_to_cos:
+        emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1)
+
+    # zero pad
+    if embedding_dim % 2 == 1:
+        emb = torch.nn.functional.pad(emb, (0, 1, 0, 0))
+    return emb
+
+
+def apply_rotary_emb_qwen(
+    x: torch.Tensor,
+    freqs_cis: Union[torch.Tensor, tuple[torch.Tensor]],
+    use_real: bool = True,
+    use_real_unbind_dim: int = -1,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """
+    Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings
+    to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are
+    reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting
+    tensors contain rotary embeddings and are returned as real tensors.
+
+    Args:
+        x (`torch.Tensor`):
+            Query or key tensor to apply rotary embeddings. [B, S, H, D] xk (torch.Tensor): Key tensor to apply
+        freqs_cis (`tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],)
+
+    Returns:
+        tuple[torch.Tensor, torch.Tensor]: tuple of modified query tensor and key tensor with rotary embeddings.
+    """
+    if use_real:
+        cos, sin = freqs_cis  # [S, D]
+        cos = cos[None, None]
+        sin = sin[None, None]
+        cos, sin = cos.to(x.device), sin.to(x.device)
+
+        if use_real_unbind_dim == -1:
+            # Used for flux, cogvideox, hunyuan-dit
+            x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1)  # [B, S, H, D//2]
+            x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3)
+        elif use_real_unbind_dim == -2:
+            # Used for Stable Audio, OmniGen, CogView4 and Cosmos
+            x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2)  # [B, S, H, D//2]
+            x_rotated = torch.cat([-x_imag, x_real], dim=-1)
+        else:
+            raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.")
+
+        out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype)
+
+        return out
+    else:
+        x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2))
+        freqs_cis = freqs_cis.unsqueeze(1)
+        x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3)
+
+        return x_out.type_as(x)
+
+
+class QwenImagePipeline(
+    nn.Module,
+):
+    def __init__(
+        self,
+        *,
+        od_config: OmniDiffusionConfig = None,
+        prefix: str = "",
+    ):
+        super().__init__()
+        self.device = get_local_device()
+        # self.hf_config = vllm_config.model_config.hf_config
+        # TODO support user custom folders
+        self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained("Qwen/Qwen-Image", subfolder="scheduler")
+        logger.info("Loaded Qwen-Image scheduler successfully")
+        from vllm_omni.diffusion.utils.torch_utils import (
+            set_default_torch_dtype,
+        )  # to avoid ci error
+
+        with set_default_torch_dtype(torch.bfloat16):
+            self.text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained(
+                "Qwen/Qwen-Image", subfolder="text_encoder"
+            )
+            logger.info("Loaded Qwen-Image text encoder successfully")
+            self.vae = AutoencoderKLQwenImage.from_pretrained("Qwen/Qwen-Image", subfolder="vae").to(self.device)
+            logger.info("Loaded Qwen-Image VAE successfully")
+            self.transformer = QwenImageTransformer2DModel()
+            logger.info("Initialized Qwen-Image transformer successfully.")
+            self.tokenizer = Qwen2Tokenizer.from_pretrained("Qwen/Qwen-Image", subfolder="tokenizer")
+            logger.info("Loaded Qwen-Image tokenizer successfully.")
+
+        self.stage = None
+
+        self.vae_scale_factor = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8
+        # QwenImage latents are turned into 2x2 patches and packed.
+        # This means the latent width and height has to be divisible
+        # by the patch size. So the vae scale factor is multiplied by the patch size to account for this
+        # self.image_processor = VaeImageProcessor(
+        #     vae_scale_factor=self.vae_scale_factor * 2
+        # )
+        self.tokenizer_max_length = 1024
+        self.prompt_template_encode = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"  # noqa: E501
+        self.prompt_template_encode_start_idx = 34
+        self.default_sample_size = 128
+
+    def check_inputs(
+        self,
+        prompt,
+        height,
+        width,
+        negative_prompt=None,
+        prompt_embeds=None,
+        negative_prompt_embeds=None,
+        prompt_embeds_mask=None,
+        negative_prompt_embeds_mask=None,
+        callback_on_step_end_tensor_inputs=None,
+        max_sequence_length=None,
+    ):
+        if height % (self.vae_scale_factor * 2) != 0 or width % (self.vae_scale_factor * 2) != 0:
+            logger.warning(
+                f"`height` and `width` have to be divisible by {self.vae_scale_factor * 2} "
+                "but are {height} and {width}. Dimensions will be resized accordingly"
+            )
+
+        # if callback_on_step_end_tensor_inputs is not None and not all(
+        #     k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
+        # ):
+        #     raise ValueError(
+        #         f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs},
+        # but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
+        #     )
+
+        if prompt is not None and prompt_embeds is not None:
+            raise ValueError(
+                f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
+                " only forward one of the two."
+            )
+        elif prompt is None and prompt_embeds is None:
+            raise ValueError(
+                "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
+            )
+        elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
+            raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
+
+        if negative_prompt is not None and negative_prompt_embeds is not None:
+            raise ValueError(
+                f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
+                f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
+            )
+
+        if prompt_embeds is not None and prompt_embeds_mask is None:
+            raise ValueError(
+                "If `prompt_embeds` are provided, `prompt_embeds_mask` also have to be passed. "
+                "Make sure to generate `prompt_embeds_mask` from the same text encoder "
+                "that was used to generate `prompt_embeds`."
+            )
+        if negative_prompt_embeds is not None and negative_prompt_embeds_mask is None:
+            raise ValueError(
+                "If `negative_prompt_embeds` are provided, `negative_prompt_embeds_mask` also have to be passed. "
+                "Make sure to generate `negative_prompt_embeds_mask` from the same text encoder "
+                "that was used to generate `negative_prompt_embeds`."
+            )
+
+        if max_sequence_length is not None and max_sequence_length > 1024:
+            raise ValueError(f"`max_sequence_length` cannot be greater than 1024 but is {max_sequence_length}")
+
+    def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor):
+        bool_mask = mask.bool()
+        valid_lengths = bool_mask.sum(dim=1)
+        selected = hidden_states[bool_mask]
+        split_result = torch.split(selected, valid_lengths.tolist(), dim=0)
+
+        return split_result
+
+    def _get_qwen_prompt_embeds(
+        self,
+        prompt: Union[str, list[str]] = None,
+        dtype: Optional[torch.dtype] = None,
+    ):
+        dtype = dtype or self.text_encoder.dtype
+
+        prompt = [prompt] if isinstance(prompt, str) else prompt
+
+        template = self.prompt_template_encode
+        drop_idx = self.prompt_template_encode_start_idx
+        txt = [template.format(e) for e in prompt]
+        txt_tokens = self.tokenizer(
+            txt,
+            max_length=self.tokenizer_max_length + drop_idx,
+            padding=True,
+            truncation=True,
+            return_tensors="pt",
+        ).to(self.device)
+        # print(f"attention mask: {txt_tokens.attention_mask}")
+        encoder_hidden_states = self.text_encoder(
+            input_ids=txt_tokens.input_ids,
+            attention_mask=txt_tokens.attention_mask,
+            output_hidden_states=True,
+        )
+        hidden_states = encoder_hidden_states.hidden_states[-1]
+        split_hidden_states = self._extract_masked_hidden(hidden_states, txt_tokens.attention_mask)
+        split_hidden_states = [e[drop_idx:] for e in split_hidden_states]
+        attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states]
+        max_seq_len = max([e.size(0) for e in split_hidden_states])
+        prompt_embeds = torch.stack(
+            [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states]
+        )
+        encoder_attention_mask = torch.stack(
+            [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list]
+        )
+
+        prompt_embeds = prompt_embeds.to(dtype=dtype)
+
+        return prompt_embeds, encoder_attention_mask
+
+    def encode_prompt(
+        self,
+        prompt: Union[str, list[str]],
+        num_images_per_prompt: int = 1,
+        prompt_embeds: Optional[torch.Tensor] = None,
+        prompt_embeds_mask: Optional[torch.Tensor] = None,
+        max_sequence_length: int = 1024,
+    ):
+        r"""
+
+        Args:
+            prompt (`str` or `list[str]`, *optional*):
+                prompt to be encoded
+            device: (`torch.device`):
+                torch device
+            num_images_per_prompt (`int`):
+                number of images that should be generated per prompt
+            prompt_embeds (`torch.Tensor`, *optional*):
+                Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
+                provided, text embeddings will be generated from `prompt` input argument.
+        """
+
+        prompt = [prompt] if isinstance(prompt, str) else prompt
+        batch_size = len(prompt) if prompt_embeds is None else prompt_embeds.shape[0]
+
+        if prompt_embeds is None:
+            prompt_embeds, prompt_embeds_mask = self._get_qwen_prompt_embeds(prompt)
+
+        prompt_embeds = prompt_embeds[:, :max_sequence_length]
+        prompt_embeds_mask = prompt_embeds_mask[:, :max_sequence_length]
+
+        _, seq_len, _ = prompt_embeds.shape
+        prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
+        prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
+        prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1)
+        prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len)
+
+        return prompt_embeds, prompt_embeds_mask
+
+    @staticmethod
+    def _pack_latents(latents, batch_size, num_channels_latents, height, width):
+        latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2)
+        latents = latents.permute(0, 2, 4, 1, 3, 5)
+        latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4)
+
+        return latents
+
+    @staticmethod
+    def _unpack_latents(latents, height, width, vae_scale_factor):
+        batch_size, num_patches, channels = latents.shape
+
+        # VAE applies 8x compression on images but we must also account for packing which requires
+        # latent height and width to be divisible by 2.
+        height = 2 * (int(height) // (vae_scale_factor * 2))
+        width = 2 * (int(width) // (vae_scale_factor * 2))
+
+        latents = latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2)
+        latents = latents.permute(0, 3, 1, 4, 2, 5)
+
+        latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width)
+
+        return latents
+
+    def prepare_latents(
+        self,
+        batch_size,
+        num_channels_latents,
+        height,
+        width,
+        dtype,
+        device,
+        generator,
+        latents=None,
+    ) -> torch.Tensor:
+        # generator=torch.Generator(device="cuda").manual_seed(42)
+        # VAE applies 8x compression on images but we must also account for packing which requires
+        # latent height and width to be divisible by 2.
+        height = 2 * (int(height) // (self.vae_scale_factor * 2))
+        width = 2 * (int(width) // (self.vae_scale_factor * 2))
+
+        shape = (batch_size, 1, num_channels_latents, height, width)
+
+        if latents is not None:
+            return latents.to(device=device, dtype=dtype)
+
+        if isinstance(generator, list) and len(generator) != batch_size:
+            raise ValueError(
+                f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
+                f" size of {batch_size}. Make sure the batch size matches the length of the generators."
+            )
+
+        latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
+        latents = self._pack_latents(latents, batch_size, num_channels_latents, height, width)
+
+        return latents
+
+    def prepare_timesteps(self, num_inference_steps, sigmas, image_seq_len):
+        sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas
+        # image_seq_len = latents.shape[1]
+        mu = calculate_shift(
+            image_seq_len,
+            self.scheduler.config.get("base_image_seq_len", 256),
+            self.scheduler.config.get("max_image_seq_len", 4096),
+            self.scheduler.config.get("base_shift", 0.5),
+            self.scheduler.config.get("max_shift", 1.15),
+        )
+        timesteps, num_inference_steps = retrieve_timesteps(
+            self.scheduler,
+            num_inference_steps,
+            sigmas=sigmas,
+            mu=mu,
+        )
+        return timesteps, num_inference_steps
+
+    @property
+    def guidance_scale(self):
+        return self._guidance_scale
+
+    @property
+    def attention_kwargs(self):
+        return self._attention_kwargs
+
+    @property
+    def num_timesteps(self):
+        return self._num_timesteps
+
+    @property
+    def current_timestep(self):
+        return self._current_timestep
+
+    @property
+    def interrupt(self):
+        return self._interrupt
+
+    def diffuse(
+        self,
+        prompt_embeds,
+        prompt_embeds_mask,
+        negative_prompt_embeds,
+        negative_prompt_embeds_mask,
+        latents,
+        img_shapes,
+        txt_seq_lens,
+        negative_txt_seq_lens,
+        timesteps,
+        do_true_cfg,
+        guidance,
+        true_cfg_scale,
+    ):
+        self.scheduler.set_begin_index(0)
+        for i, t in enumerate(timesteps):
+            if self.interrupt:
+                continue
+            self._current_timestep = t
+            # broadcast to batch dimension and place on same device/dtype as latents
+            timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
+            noise_pred = self.transformer(
+                hidden_states=latents,
+                timestep=timestep / 1000,
+                guidance=guidance,
+                encoder_hidden_states_mask=prompt_embeds_mask,
+                encoder_hidden_states=prompt_embeds,
+                img_shapes=img_shapes,
+                txt_seq_lens=txt_seq_lens,
+                attention_kwargs=self.attention_kwargs,
+                return_dict=False,
+            )[0]
+            if do_true_cfg:
+                neg_noise_pred = self.transformer(
+                    hidden_states=latents,
+                    timestep=timestep / 1000,
+                    guidance=guidance,
+                    encoder_hidden_states_mask=negative_prompt_embeds_mask,
+                    encoder_hidden_states=negative_prompt_embeds,
+                    img_shapes=img_shapes,
+                    txt_seq_lens=negative_txt_seq_lens,
+                    attention_kwargs=self.attention_kwargs,
+                    return_dict=False,
+                )[0]
+                comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred)
+                cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
+                noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
+                noise_pred = comb_pred * (cond_norm / noise_norm)
+            # compute the previous noisy sample x_t -> x_t-1
+            latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
+        return latents
+
+    def forward(
+        self,
+        req: OmniDiffusionRequest,
+        prompt: Union[str, list[str]] = "",
+        negative_prompt: Union[str, list[str]] = "",
+        true_cfg_scale: float = 4.0,
+        height: int | None = None,
+        width: int | None = None,
+        num_inference_steps: int = 50,
+        sigmas: Optional[list[float]] = None,
+        guidance_scale: float = 1.0,
+        num_images_per_prompt: int = 1,
+        generator: Optional[Union[torch.Generator, list[torch.Generator]]] = None,
+        latents: Optional[torch.Tensor] = None,
+        prompt_embeds: Optional[torch.Tensor] = None,
+        prompt_embeds_mask: Optional[torch.Tensor] = None,
+        negative_prompt_embeds: Optional[torch.Tensor] = None,
+        negative_prompt_embeds_mask: Optional[torch.Tensor] = None,
+        output_type: Optional[str] = "pil",
+        attention_kwargs: Optional[dict[str, Any]] = None,
+        callback_on_step_end_tensor_inputs: list[str] = ["latents"],
+        max_sequence_length: int = 512,
+    ) -> DiffusionOutput:
+        # # TODO: only support single prompt now
+        # if req.prompt is not None:
+        #     prompt = req.prompt[0] if isinstance(req.prompt, list) else req.prompt
+        prompt = req.prompt if req.prompt is not None else prompt
+        negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
+        height = req.height or self.default_sample_size * self.vae_scale_factor
+        width = req.width or self.default_sample_size * self.vae_scale_factor
+        num_inference_steps = req.num_inference_steps or num_inference_steps
+        generator = req.generator or generator
+        true_cfg_scale = req.true_cfg_scale or true_cfg_scale
+        # 1. check inputs
+        # 2. encode prompts
+        # 3. prepare latents and timesteps
+        # 4. diffusion process
+        # 5. decode latents
+        # 6. post-process outputs
+        self.check_inputs(
+            prompt,
+            height,
+            width,
+            negative_prompt,
+            prompt_embeds,
+            negative_prompt_embeds,
+            prompt_embeds_mask,
+            negative_prompt_embeds_mask,
+            callback_on_step_end_tensor_inputs,
+            max_sequence_length,
+        )
+
+        self._guidance_scale = guidance_scale
+        self._attention_kwargs = attention_kwargs
+        self._current_timestep = None
+        self._interrupt = False
+
+        if prompt is not None and isinstance(prompt, str):
+            batch_size = 1
+        elif prompt is not None and isinstance(prompt, list):
+            batch_size = len(prompt)
+        else:
+            batch_size = prompt_embeds.shape[0]
+
+        has_neg_prompt = negative_prompt is not None or (
+            negative_prompt_embeds is not None and negative_prompt_embeds_mask is not None
+        )
+
+        do_true_cfg = true_cfg_scale > 1 and has_neg_prompt
+        prompt_embeds, prompt_embeds_mask = self.encode_prompt(
+            prompt=prompt,
+            prompt_embeds=prompt_embeds,
+            prompt_embeds_mask=prompt_embeds_mask,
+            num_images_per_prompt=num_images_per_prompt,
+            max_sequence_length=max_sequence_length,
+        )
+        if do_true_cfg:
+            negative_prompt_embeds, negative_prompt_embeds_mask = self.encode_prompt(
+                prompt=negative_prompt,
+                prompt_embeds=negative_prompt_embeds,
+                prompt_embeds_mask=negative_prompt_embeds_mask,
+                num_images_per_prompt=num_images_per_prompt,
+                max_sequence_length=max_sequence_length,
+            )
+
+        num_channels_latents = self.transformer.in_channels // 4
+        latents = self.prepare_latents(
+            batch_size * num_images_per_prompt,
+            num_channels_latents,
+            height,
+            width,
+            prompt_embeds.dtype,
+            self.device,
+            generator,
+            latents,
+        )
+        img_shapes = [
+            [
+                (
+                    1,
+                    height // self.vae_scale_factor // 2,
+                    width // self.vae_scale_factor // 2,
+                )
+            ]
+        ] * batch_size
+
+        timesteps, num_inference_steps = self.prepare_timesteps(num_inference_steps, sigmas, latents.shape[1])
+        # num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
+        self._num_timesteps = len(timesteps)
+
+        # handle guidance
+        if self.transformer.guidance_embeds:
+            guidance = torch.full([1], guidance_scale, dtype=torch.float32)
+            guidance = guidance.expand(latents.shape[0])
+        else:
+            guidance = None
+
+        if self.attention_kwargs is None:
+            self._attention_kwargs = {}
+
+        txt_seq_lens = prompt_embeds_mask.sum(dim=1).tolist() if prompt_embeds_mask is not None else None
+        negative_txt_seq_lens = (
+            negative_prompt_embeds_mask.sum(dim=1).tolist() if negative_prompt_embeds_mask is not None else None
+        )
+        # print inputp params
+
+        latents = self.diffuse(
+            prompt_embeds,
+            prompt_embeds_mask,
+            negative_prompt_embeds,
+            negative_prompt_embeds_mask,
+            latents,
+            img_shapes,
+            txt_seq_lens,
+            negative_txt_seq_lens,
+            timesteps,
+            do_true_cfg,
+            guidance,
+            true_cfg_scale,
+        )
+
+        self._current_timestep = None
+        if output_type == "latent":
+            image = latents
+        else:
+            latents = self._unpack_latents(latents, height, width, self.vae_scale_factor)
+            latents = latents.to(self.vae.dtype)
+            latents_mean = (
+                torch.tensor(self.vae.config.latents_mean)
+                .view(1, self.vae.config.z_dim, 1, 1, 1)
+                .to(latents.device, latents.dtype)
+            )
+            latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
+                latents.device, latents.dtype
+            )
+            latents = latents / latents_std + latents_mean
+            image = self.vae.decode(latents, return_dict=False)[0][:, :, 0]
+            # processed_image = self.image_processor.postprocess(image, output_type=output_type)
+
+        return DiffusionOutput(output=image)
+
+    def load_weights(self):
+        self.load_transformer()
+
+    # TODO: find a better way to load
+    def load_transformer(self):
+        import glob
+        import os
+
+        # Define the weight iterator
+        def weight_iterator(transformer_path):
+            if not os.path.exists(transformer_path):
+                logger.warning(f"Path {transformer_path} does not exist.")
+                return
+
+            # Look for safetensors first
+            safetensors_files = glob.glob(os.path.join(transformer_path, "*.safetensors"))
+            if safetensors_files:
+                try:
+                    from safetensors.torch import load_file
+                except ImportError:
+                    logger.warning("safetensors not installed, cannot load .safetensors files.")
+                    return
+
+                for file_path in safetensors_files:
+                    state_dict = load_file(file_path)
+                    for name, tensor in state_dict.items():
+                        yield name, tensor
+            else:
+                # Fallback to bin
+                bin_files = glob.glob(os.path.join(transformer_path, "*.bin"))
+                for file_path in bin_files:
+                    state_dict = torch.load(file_path)
+                    for name, tensor in state_dict.items():
+                        yield name, tensor
+
+        model_name = "Qwen/Qwen-Image"
+        try:
+            # Use the arguments as verified by the user
+            model_path = download_weights_from_hf_specific(model_name, None, ["*"])
+            transformer_path = os.path.join(model_path, "transformer")
+            self.transformer.load_weights(weight_iterator(transformer_path))
+
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            raise e

vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py

@@ -0,0 +1,700 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import functools
+from collections.abc import Iterable
+from typing import Any, Optional, Union
+
+import torch
+import torch.nn as nn
+from diffusers.models.attention import FeedForward
+from diffusers.models.attention_dispatch import dispatch_attention_fn
+
+# TODO replace this with vLLM implementation
+from diffusers.models.embeddings import TimestepEmbedding, Timesteps
+from diffusers.models.modeling_outputs import Transformer2DModelOutput
+from diffusers.models.normalization import AdaLayerNormContinuous
+from vllm.logger import init_logger
+from vllm.model_executor.layers.layernorm import RMSNorm
+from vllm.model_executor.layers.linear import QKVParallelLinear, ReplicatedLinear
+
+logger = init_logger(__name__)
+
+
+def apply_rotary_emb_qwen(
+    x: torch.Tensor,
+    freqs_cis: Union[torch.Tensor, tuple[torch.Tensor]],
+    use_real: bool = True,
+    use_real_unbind_dim: int = -1,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """
+    Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings
+    to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are
+    reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting
+    tensors contain rotary embeddings and are returned as real tensors.
+
+    Args:
+        x (`torch.Tensor`):
+            Query or key tensor to apply rotary embeddings. [B, S, H, D] xk (torch.Tensor): Key tensor to apply
+        freqs_cis (`tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],)
+
+    Returns:
+        tuple[torch.Tensor, torch.Tensor]: tuple of modified query tensor and key tensor with rotary embeddings.
+    """
+    if use_real:
+        cos, sin = freqs_cis  # [S, D]
+        cos = cos[None, None]
+        sin = sin[None, None]
+        cos, sin = cos.to(x.device), sin.to(x.device)
+
+        if use_real_unbind_dim == -1:
+            # Used for flux, cogvideox, hunyuan-dit
+            x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1)  # [B, S, H, D//2]
+            x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3)
+        elif use_real_unbind_dim == -2:
+            # Used for Stable Audio, OmniGen, CogView4 and Cosmos
+            x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2)  # [B, S, H, D//2]
+            x_rotated = torch.cat([-x_imag, x_real], dim=-1)
+        else:
+            raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.")
+
+        out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype)
+
+        return out
+    else:
+        x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2))
+        freqs_cis = freqs_cis.unsqueeze(1)
+        x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3)
+
+        return x_out.type_as(x)
+
+
+class QwenTimestepProjEmbeddings(nn.Module):
+    def __init__(self, embedding_dim):
+        super().__init__()
+
+        self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0, scale=1000)
+        self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim)
+
+    def forward(self, timestep, hidden_states):
+        timesteps_proj = self.time_proj(timestep)
+        timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_states.dtype))  # (N, D)
+
+        conditioning = timesteps_emb
+
+        return conditioning
+
+
+class QwenEmbedRope(nn.Module):
+    def __init__(self, theta: int, axes_dim: list[int], scale_rope=False):
+        super().__init__()
+        self.theta = theta
+        self.axes_dim = axes_dim
+        pos_index = torch.arange(4096)
+        neg_index = torch.arange(4096).flip(0) * -1 - 1
+        self.pos_freqs = torch.cat(
+            [
+                self.rope_params(pos_index, self.axes_dim[0], self.theta),
+                self.rope_params(pos_index, self.axes_dim[1], self.theta),
+                self.rope_params(pos_index, self.axes_dim[2], self.theta),
+            ],
+            dim=1,
+        )
+        self.neg_freqs = torch.cat(
+            [
+                self.rope_params(neg_index, self.axes_dim[0], self.theta),
+                self.rope_params(neg_index, self.axes_dim[1], self.theta),
+                self.rope_params(neg_index, self.axes_dim[2], self.theta),
+            ],
+            dim=1,
+        )
+        self.rope_cache = {}
+
+        # DO NOT USING REGISTER BUFFER HERE, IT WILL CAUSE COMPLEX NUMBERS LOSE ITS IMAGINARY PART
+        self.scale_rope = scale_rope
+
+    def rope_params(self, index, dim, theta=10000):
+        """
+        Args:
+            index: [0, 1, 2, 3] 1D Tensor representing the position index of the token
+        """
+        assert dim % 2 == 0
+        freqs = torch.outer(
+            index,
+            1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float32).div(dim)),
+        )
+        freqs = torch.polar(torch.ones_like(freqs), freqs)
+        return freqs
+
+    def forward(self, video_fhw, txt_seq_lens, device):
+        """
+        Args: video_fhw: [frame, height, width] a list of 3 integers representing the shape of the video Args:
+        txt_length: [bs] a list of 1 integers representing the length of the text
+        """
+        if self.pos_freqs.device != device:
+            self.pos_freqs = self.pos_freqs.to(device)
+            self.neg_freqs = self.neg_freqs.to(device)
+
+        if isinstance(video_fhw, list):
+            video_fhw = video_fhw[0]
+        if not isinstance(video_fhw, list):
+            video_fhw = [video_fhw]
+
+        vid_freqs = []
+        max_vid_index = 0
+        for idx, fhw in enumerate(video_fhw):
+            frame, height, width = fhw
+            rope_key = f"{idx}_{height}_{width}"
+
+            if not torch.compiler.is_compiling():
+                if rope_key not in self.rope_cache:
+                    self.rope_cache[rope_key] = self._compute_video_freqs(frame, height, width, idx)
+                video_freq = self.rope_cache[rope_key]
+            else:
+                video_freq = self._compute_video_freqs(frame, height, width, idx)
+            video_freq = video_freq.to(device)
+            vid_freqs.append(video_freq)
+
+            if self.scale_rope:
+                max_vid_index = max(height // 2, width // 2, max_vid_index)
+            else:
+                max_vid_index = max(height, width, max_vid_index)
+
+        max_len = max(txt_seq_lens)
+        txt_freqs = self.pos_freqs[max_vid_index : max_vid_index + max_len, ...]
+        vid_freqs = torch.cat(vid_freqs, dim=0)
+
+        return vid_freqs, txt_freqs
+
+    @functools.cache
+    def _compute_video_freqs(self, frame, height, width, idx=0):
+        seq_lens = frame * height * width
+        freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1)
+        freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1)
+
+        freqs_frame = freqs_pos[0][idx : idx + frame].view(frame, 1, 1, -1).expand(frame, height, width, -1)
+        if self.scale_rope:
+            freqs_height = torch.cat(
+                [freqs_neg[1][-(height - height // 2) :], freqs_pos[1][: height // 2]],
+                dim=0,
+            )
+            freqs_height = freqs_height.view(1, height, 1, -1).expand(frame, height, width, -1)
+            freqs_width = torch.cat(
+                [freqs_neg[2][-(width - width // 2) :], freqs_pos[2][: width // 2]],
+                dim=0,
+            )
+            freqs_width = freqs_width.view(1, 1, width, -1).expand(frame, height, width, -1)
+        else:
+            freqs_height = freqs_pos[1][:height].view(1, height, 1, -1).expand(frame, height, width, -1)
+            freqs_width = freqs_pos[2][:width].view(1, 1, width, -1).expand(frame, height, width, -1)
+
+        freqs = torch.cat([freqs_frame, freqs_height, freqs_width], dim=-1).reshape(seq_lens, -1)
+        return freqs.clone().contiguous()
+
+
+class QwenImageCrossAttention(nn.Module):
+    def __init__(
+        self,
+        dim: int,  # query_dim
+        num_heads: int,
+        head_dim: int,
+        window_size=(-1, -1),
+        added_kv_proj_dim: int = None,
+        out_bias: bool = True,
+        qk_norm=True,  # rmsnorm
+        eps=1e-6,
+        pre_only=False,
+        context_pre_only: bool = False,
+        parallel_attention=False,
+        out_dim: int = None,
+    ) -> None:
+        assert dim % num_heads == 0
+        super().__init__()
+        self.dim = dim
+        self.num_heads = num_heads
+        self.head_dim = dim // num_heads
+        self.window_size = window_size
+        self.qk_norm = qk_norm
+        self.eps = eps
+        self.parallel_attention = parallel_attention
+
+        # layers
+        # self.to_q = ReplicatedLinear(dim, dim)
+        # self.to_k = ReplicatedLinear(dim, dim)
+        # self.to_v = ReplicatedLinear(dim, dim)
+        self.to_qkv = QKVParallelLinear(
+            hidden_size=dim,
+            head_size=self.head_dim,
+            total_num_heads=num_heads,
+            disable_tp=True,
+        )
+        self.norm_q = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()
+        self.norm_k = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()
+        self.inner_dim = out_dim if out_dim is not None else head_dim * num_heads
+        self.inner_kv_dim = self.inner_dim
+        if added_kv_proj_dim is not None:
+            assert context_pre_only is not None
+            # self.add_k_proj = ReplicatedLinear(added_kv_proj_dim, self.inner_kv_dim, bias=True)
+            # self.add_v_proj = ReplicatedLinear(added_kv_proj_dim, self.inner_kv_dim, bias=True)
+            # self.add_q_proj = ReplicatedLinear(
+            #     added_kv_proj_dim, self.inner_dim, bias=True
+            # )
+            self.add_kv_proj = QKVParallelLinear(
+                added_kv_proj_dim,
+                head_size=self.inner_kv_dim // self.num_heads,
+                total_num_heads=self.num_heads,
+                disable_tp=True,
+            )
+
+        if context_pre_only is not None and not context_pre_only:
+            self.to_add_out = ReplicatedLinear(self.inner_dim, self.dim, bias=out_bias)
+        else:
+            self.to_add_out = None
+
+        if not pre_only:
+            self.to_out = nn.ModuleList([])
+            self.to_out.append(ReplicatedLinear(self.inner_dim, self.dim, bias=out_bias))
+        else:
+            self.to_out = None
+
+        self.norm_added_q = RMSNorm(head_dim, eps=eps)
+        self.norm_added_k = RMSNorm(head_dim, eps=eps)
+
+        # self.attn = Attention(
+        #     num_heads=num_heads,
+        #     head_size=self.head_dim,
+        #     scale=1.0,  # softmax_scale
+        #     attn_type=AttentionType.ENCODER,
+        # )
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        encoder_hidden_states: torch.Tensor,
+        image_rotary_emb: tuple[torch.Tensor, torch.Tensor],
+        **cross_attention_kwargs,
+    ):
+        seq_len_txt = encoder_hidden_states.shape[1]
+
+        # Compute QKV for image stream (sample projections)
+        qkv, _ = self.to_qkv(hidden_states)
+        img_query, img_key, img_value = qkv.chunk(3, dim=-1)
+
+        # Compute QKV for text stream (context projections)
+        qkv, _ = self.add_kv_proj(encoder_hidden_states)
+        txt_query, txt_key, txt_value = qkv.chunk(3, dim=-1)
+
+        # Reshape for multi-head attention
+        img_query = img_query.unflatten(-1, (self.num_heads, -1))
+        img_key = img_key.unflatten(-1, (self.num_heads, -1))
+        img_value = img_value.unflatten(-1, (self.num_heads, -1))
+
+        txt_query = txt_query.unflatten(-1, (self.num_heads, -1))
+        txt_key = txt_key.unflatten(-1, (self.num_heads, -1))
+        txt_value = txt_value.unflatten(-1, (self.num_heads, -1))
+
+        # Apply QK normalization
+        img_query = self.norm_q(img_query)
+        img_key = self.norm_k(img_key)
+        txt_query = self.norm_added_q(txt_query)
+        txt_key = self.norm_added_k(txt_key)
+
+        # Apply RoPE
+        if image_rotary_emb is not None:
+            img_freqs, txt_freqs = image_rotary_emb
+            img_query = apply_rotary_emb_qwen(img_query, img_freqs, use_real=False)
+            img_key = apply_rotary_emb_qwen(img_key, img_freqs, use_real=False)
+            txt_query = apply_rotary_emb_qwen(txt_query, txt_freqs, use_real=False)
+            txt_key = apply_rotary_emb_qwen(txt_key, txt_freqs, use_real=False)
+
+        # Concatenate for joint attention
+        # Order: [text, image]
+        joint_query = torch.cat([txt_query, img_query], dim=1)
+        joint_key = torch.cat([txt_key, img_key], dim=1)
+        joint_value = torch.cat([txt_value, img_value], dim=1)
+
+        # Compute joint attention
+        # joint_hidden_states = self.attn(
+        #     joint_query,
+        #     joint_key,
+        #     joint_value,
+        # )
+        joint_hidden_states = dispatch_attention_fn(
+            joint_query,
+            joint_key,
+            joint_value,
+            dropout_p=0.0,
+            is_causal=False,
+        )
+
+        # Reshape back
+        joint_hidden_states = joint_hidden_states.flatten(2, 3)
+        joint_hidden_states = joint_hidden_states.to(joint_query.dtype)
+
+        # Split attention outputs back
+        txt_attn_output = joint_hidden_states[:, :seq_len_txt, :]  # Text part
+        img_attn_output = joint_hidden_states[:, seq_len_txt:, :]  # Image part
+
+        # Apply output projections
+        img_attn_output, _ = self.to_out[0](img_attn_output)
+        if len(self.to_out) > 1:
+            (img_attn_output,) = self.to_out[1](img_attn_output)  # dropout
+
+        txt_attn_output, _ = self.to_add_out(txt_attn_output)
+
+        return img_attn_output, txt_attn_output
+
+
+class QwenImageTransformerBlock(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_attention_heads: int,
+        attention_head_dim: int,
+        qk_norm: str = "rms_norm",
+        eps: float = 1e-6,
+    ):
+        super().__init__()
+
+        self.dim = dim
+        self.num_attention_heads = num_attention_heads
+        self.attention_head_dim = attention_head_dim
+
+        # Image processing modules
+        self.img_mod = nn.Sequential(
+            nn.SiLU(),
+            nn.Linear(dim, 6 * dim, bias=True),  # For scale, shift, gate for norm1 and norm2
+        )
+        self.img_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
+        # self.attn = Attention(
+        #     query_dim=dim,
+        #     cross_attention_dim=None,  # Enable cross attention for joint computation
+        #     added_kv_proj_dim=dim,  # Enable added KV projections for text stream
+        #     dim_head=attention_head_dim,
+        #     heads=num_attention_heads,
+        #     out_dim=dim,
+        #     context_pre_only=False,
+        #     bias=True,
+        #     processor=QwenDoubleStreamAttnProcessor2_0(),
+        #     qk_norm=qk_norm,
+        #     eps=eps,
+        # )
+        self.attn = QwenImageCrossAttention(
+            dim=dim,
+            num_heads=num_attention_heads,
+            added_kv_proj_dim=dim,
+            context_pre_only=False,
+            head_dim=attention_head_dim,
+        )
+        self.img_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
+        self.img_mlp = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate")
+
+        # Text processing modules
+        self.txt_mod = nn.Sequential(
+            nn.SiLU(),
+            nn.Linear(dim, 6 * dim, bias=True),  # For scale, shift, gate for norm1 and norm2
+        )
+        self.txt_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
+        # Text doesn't need separate attention - it's handled by img_attn joint computation
+        self.txt_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps)
+        self.txt_mlp = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate")
+
+    def _modulate(self, x, mod_params):
+        """Apply modulation to input tensor"""
+        shift, scale, gate = mod_params.chunk(3, dim=-1)
+        return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1), gate.unsqueeze(1)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        encoder_hidden_states: torch.Tensor,
+        encoder_hidden_states_mask: torch.Tensor,
+        temb: torch.Tensor,
+        image_rotary_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
+        joint_attention_kwargs: Optional[dict[str, Any]] = None,
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        # Get modulation parameters for both streams
+        img_mod_params = self.img_mod(temb)  # [B, 6*dim]
+        txt_mod_params = self.txt_mod(temb)  # [B, 6*dim]
+
+        # Split modulation parameters for norm1 and norm2
+        img_mod1, img_mod2 = img_mod_params.chunk(2, dim=-1)  # Each [B, 3*dim]
+        txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1)  # Each [B, 3*dim]
+
+        # Process image stream - norm1 + modulation
+        img_normed = self.img_norm1(hidden_states)
+        img_modulated, img_gate1 = self._modulate(img_normed, img_mod1)
+
+        # Process text stream - norm1 + modulation
+        txt_normed = self.txt_norm1(encoder_hidden_states)
+        txt_modulated, txt_gate1 = self._modulate(txt_normed, txt_mod1)
+
+        # Use QwenAttnProcessor2_0 for joint attention computation
+        # This directly implements the DoubleStreamLayerMegatron logic:
+        # 1. Computes QKV for both streams
+        # 2. Applies QK normalization and RoPE
+        # 3. Concatenates and runs joint attention
+        # 4. Splits results back to separate streams
+        joint_attention_kwargs = joint_attention_kwargs or {}
+        attn_output = self.attn(
+            hidden_states=img_modulated,  # Image stream (will be processed as "sample")
+            encoder_hidden_states=txt_modulated,  # Text stream (will be processed as "context")
+            encoder_hidden_states_mask=encoder_hidden_states_mask,
+            image_rotary_emb=image_rotary_emb,
+            **joint_attention_kwargs,
+        )
+
+        # QwenAttnProcessor2_0 returns (img_output, txt_output) when encoder_hidden_states is provided
+        img_attn_output, txt_attn_output = attn_output
+
+        # Apply attention gates and add residual (like in Megatron)
+        hidden_states = hidden_states + img_gate1 * img_attn_output
+        encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output
+
+        # Process image stream - norm2 + MLP
+        img_normed2 = self.img_norm2(hidden_states)
+        img_modulated2, img_gate2 = self._modulate(img_normed2, img_mod2)
+        img_mlp_output = self.img_mlp(img_modulated2)
+        hidden_states = hidden_states + img_gate2 * img_mlp_output
+
+        # Process text stream - norm2 + MLP
+        txt_normed2 = self.txt_norm2(encoder_hidden_states)
+        txt_modulated2, txt_gate2 = self._modulate(txt_normed2, txt_mod2)
+        txt_mlp_output = self.txt_mlp(txt_modulated2)
+        encoder_hidden_states = encoder_hidden_states + txt_gate2 * txt_mlp_output
+
+        # Clip to prevent overflow for fp16
+        if encoder_hidden_states.dtype == torch.float16:
+            encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504)
+        if hidden_states.dtype == torch.float16:
+            hidden_states = hidden_states.clip(-65504, 65504)
+
+        return encoder_hidden_states, hidden_states
+
+
+class QwenImageTransformer2DModel(nn.Module):
+    """
+    The Transformer model introduced in Qwen.
+
+    Args:
+        patch_size (`int`, defaults to `2`):
+            Patch size to turn the input data into small patches.
+        in_channels (`int`, defaults to `64`):
+            The number of channels in the input.
+        out_channels (`int`, *optional*, defaults to `None`):
+            The number of channels in the output. If not specified, it defaults to `in_channels`.
+        num_layers (`int`, defaults to `60`):
+            The number of layers of dual stream DiT blocks to use.
+        attention_head_dim (`int`, defaults to `128`):
+            The number of dimensions to use for each attention head.
+        num_attention_heads (`int`, defaults to `24`):
+            The number of attention heads to use.
+        joint_attention_dim (`int`, defaults to `3584`):
+            The number of dimensions to use for the joint attention (embedding/channel dimension of
+            `encoder_hidden_states`).
+        guidance_embeds (`bool`, defaults to `False`):
+            Whether to use guidance embeddings for guidance-distilled variant of the model.
+        axes_dims_rope (`tuple[int]`, defaults to `(16, 56, 56)`):
+            The dimensions to use for the rotary positional embeddings.
+    """
+
+    # _supports_gradient_checkpointing = True
+    # _no_split_modules = ["QwenImageTransformerBlock"]
+    # _skip_layerwise_casting_patterns = ["pos_embed", "norm"]
+    # _repeated_blocks = ["QwenImageTransformerBlock"]
+    def __init__(
+        self,
+        patch_size: int = 2,
+        in_channels: int = 64,
+        out_channels: Optional[int] = 16,
+        num_layers: int = 60,
+        attention_head_dim: int = 128,
+        num_attention_heads: int = 24,
+        joint_attention_dim: int = 3584,
+        guidance_embeds: bool = False,  # TODO: this should probably be removed
+        axes_dims_rope: tuple[int, int, int] = (16, 56, 56),
+    ):
+        super().__init__()
+        self.config = None
+        self.in_channels = in_channels
+        self.out_channels = out_channels or in_channels
+        self.inner_dim = num_attention_heads * attention_head_dim
+        self.guidance_embeds = guidance_embeds
+
+        self.pos_embed = QwenEmbedRope(theta=10000, axes_dim=list(axes_dims_rope), scale_rope=True)
+
+        self.time_text_embed = QwenTimestepProjEmbeddings(embedding_dim=self.inner_dim)
+
+        self.txt_norm = RMSNorm(joint_attention_dim, eps=1e-6)
+
+        self.img_in = nn.Linear(in_channels, self.inner_dim)
+        self.txt_in = nn.Linear(joint_attention_dim, self.inner_dim)
+
+        self.transformer_blocks = nn.ModuleList(
+            [
+                QwenImageTransformerBlock(
+                    dim=self.inner_dim,
+                    num_attention_heads=num_attention_heads,
+                    attention_head_dim=attention_head_dim,
+                )
+                for _ in range(num_layers)
+            ]
+        )
+
+        self.norm_out = AdaLayerNormContinuous(self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6)
+        self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True)
+
+        self.gradient_checkpointing = False
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        encoder_hidden_states: torch.Tensor = None,
+        encoder_hidden_states_mask: torch.Tensor = None,
+        timestep: torch.LongTensor = None,
+        img_shapes: Optional[list[tuple[int, int, int]]] = None,
+        txt_seq_lens: Optional[list[int]] = None,
+        guidance: torch.Tensor = None,  # TODO: this should probably be removed
+        attention_kwargs: Optional[dict[str, Any]] = None,
+        return_dict: bool = True,
+    ) -> Union[torch.Tensor, Transformer2DModelOutput]:
+        """
+        The [`QwenTransformer2DModel`] forward method.
+
+        Args:
+            hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`):
+                Input `hidden_states`.
+            encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`):
+                Conditional embeddings (embeddings computed from the input conditions such as prompts) to use.
+            encoder_hidden_states_mask (`torch.Tensor` of shape `(batch_size, text_sequence_length)`):
+                Mask of the input conditions.
+            timestep ( `torch.LongTensor`):
+                Used to indicate denoising step.
+            attention_kwargs (`dict`, *optional*):
+                A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
+                `self.processor` in
+                [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain
+                tuple.
+
+        Returns:
+            If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a
+            `tuple` where the first element is the sample tensor.
+        """
+        # if attention_kwargs is not None:
+        #     attention_kwargs = attention_kwargs.copy()
+        #     lora_scale = attention_kwargs.pop("scale", 1.0)
+        # else:
+        #     lora_scale = 1.0
+
+        hidden_states = self.img_in(hidden_states)
+
+        # Ensure timestep tensor is on the same device and dtype as hidden_states
+        timestep = timestep.to(device=hidden_states.device, dtype=hidden_states.dtype)
+        encoder_hidden_states = self.txt_norm(encoder_hidden_states)
+        encoder_hidden_states = self.txt_in(encoder_hidden_states)
+
+        if guidance is not None:
+            guidance = guidance.to(hidden_states.dtype) * 1000
+
+        temb = (
+            self.time_text_embed(timestep, hidden_states)
+            if guidance is None
+            else self.time_text_embed(timestep, guidance, hidden_states)
+        )
+
+        image_rotary_emb = self.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device)
+
+        for index_block, block in enumerate(self.transformer_blocks):
+            encoder_hidden_states, hidden_states = block(
+                hidden_states=hidden_states,
+                encoder_hidden_states=encoder_hidden_states,
+                encoder_hidden_states_mask=encoder_hidden_states_mask,
+                temb=temb,
+                image_rotary_emb=image_rotary_emb,
+                joint_attention_kwargs=attention_kwargs,
+            )
+
+        # Use only the image part (hidden_states) from the dual-stream blocks
+        hidden_states = self.norm_out(hidden_states, temb)
+        output = self.proj_out(hidden_states)
+
+        return Transformer2DModelOutput(sample=output)
+
+    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
+        # Collect incoming weights into a dict for flexible mapping and grouping
+        weights_map: dict[str, torch.Tensor] = {name: tensor for name, tensor in weights}
+
+        stacked_params_mapping = [
+            (
+                "to_qkv",
+                [
+                    {"aliases": ["q_proj", "to_q", "q", "query"], "order": 0},
+                    {"aliases": ["k_proj", "to_k", "k", "key"], "order": 1},
+                    {"aliases": ["v_proj", "to_v", "v", "value"], "order": 2},
+                ],
+            ),
+            (
+                "add_kv_proj",
+                [
+                    {"aliases": ["add_q_proj", "add_q"], "order": 0},
+                    {"aliases": ["add_k_proj", "add_k"], "order": 1},
+                    {"aliases": ["add_v_proj", "add_v"], "order": 2},
+                ],
+            ),
+        ]
+
+        # Helper that attempts to gather shards per base key using aliases and merges them
+        for target, shards in stacked_params_mapping:
+            base_candidates: dict[str, dict[str, dict[int, tuple[str, torch.Tensor]]]] = {}
+            for key in list(weights_map.keys()):
+                for shard in shards:
+                    for alias in shard["aliases"]:
+                        weight_suffix = f".{alias}.weight"
+                        bias_suffix = f".{alias}.bias"
+                        if key.endswith(weight_suffix):
+                            base = key[: -len(weight_suffix)]
+                            base_entry = base_candidates.setdefault(base, {"weight": {}, "bias": {}})
+                            base_entry["weight"][shard["order"]] = (
+                                key,
+                                weights_map[key],
+                            )
+                            break
+                        if key.endswith(bias_suffix):
+                            base = key[: -len(bias_suffix)]
+                            base_entry = base_candidates.setdefault(base, {"weight": {}, "bias": {}})
+                            base_entry["bias"][shard["order"]] = (key, weights_map[key])
+                            break
+
+            for base, entry in base_candidates.items():
+                if len(entry["weight"]) == len(shards):
+                    ordered = [entry["weight"][order] for order in sorted(entry["weight"])]
+                    combined = torch.cat([tensor for _, tensor in ordered], dim=0)
+                    for key, _ in ordered:
+                        weights_map.pop(key, None)
+                    weights_map[base + f".{target}.weight"] = combined
+                if len(entry["bias"]) == len(shards):
+                    ordered_bias = [entry["bias"][order] for order in sorted(entry["bias"])]
+                    bcombined = torch.cat([tensor for _, tensor in ordered_bias], dim=0)
+                    for key, _ in ordered_bias:
+                        weights_map.pop(key, None)
+                    weights_map[base + f".{target}.bias"] = bcombined
+
+        # Now perform the parameter copy using the possibly-updated weights_map
+        params_dict = dict(self.named_parameters())
+        for name, loaded_weight in weights_map.items():
+            # Handle potential name mismatches for FeedForward layers
+            if name.endswith(".proj.weight") and name not in params_dict:
+                mapped_name = name.replace(".proj.weight", ".weight")
+            elif name.endswith(".proj.bias") and name not in params_dict:
+                mapped_name = name.replace(".proj.bias", ".bias")
+            else:
+                mapped_name = name
+
+            if mapped_name in params_dict:
+                param = params_dict[mapped_name]
+                with torch.no_grad():
+                    param.copy_(loaded_weight)
+            else:
+                logger.warning(f"Weight {name} (mapped to {mapped_name}) not found in model params.")

vllm_omni/diffusion/omni_diffusion.py

@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import logging
+from dataclasses import fields
+
+from vllm.logger import init_logger
+from vllm.transformers_utils.config import get_hf_file_to_dict
+
+from vllm_omni.diffusion.data import OmniDiffusionConfig
+from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
+from vllm_omni.diffusion.request import OmniDiffusionRequest
+
+# TODO configure logging properly
+logging.basicConfig(level=logging.INFO)
+
+logger = init_logger(__name__)
+
+
+def prepare_requests(prompt: str | list[str], **kwargs):
+    field_names = {f.name for f in fields(OmniDiffusionRequest)}
+
+    init_kwargs = {"prompt": prompt}
+
+    for key, value in kwargs.items():
+        if key in field_names:
+            init_kwargs[key] = value
+
+    return OmniDiffusionRequest(**init_kwargs)
+
+
+class OmniDiffusion:
+    """
+    It is the main class to interact with vLLM-Omni diffusion models.
+    It acts as a high-level interface that prepares requests and
+    delegates the actual diffusion process to the DiffusionEngine.
+
+    You can pass either an `OmniDiffusionConfig` via `od_config`, or
+    pass kwargs such as `model="Qwen/Qwen-Image"`,
+    which will be forwarded to `OmniDiffusionConfig.from_kwargs`.
+    """
+
+    def __init__(self, od_config: OmniDiffusionConfig | None = None, **kwargs):
+        if od_config is None:
+            od_config = OmniDiffusionConfig.from_kwargs(**kwargs)
+        elif isinstance(od_config, dict):
+            od_config = OmniDiffusionConfig.from_kwargs(**od_config)
+
+        self.od_config = od_config
+
+        config_dict = get_hf_file_to_dict(
+            "model_index.json",
+            od_config.model,
+        )
+        od_config.model_class_name = config_dict.get("_class_name", None)
+
+        self.engine: DiffusionEngine = DiffusionEngine.make_engine(od_config)
+
+    def generate(
+        self,
+        prompt: str | list[str],
+        **kwargs,
+    ):
+        prompts = []
+        if isinstance(prompt, str):
+            prompts.append(prompt)
+        elif isinstance(prompt, list):
+            prompts.extend(prompt)
+        else:
+            raise ValueError("Prompt must be a string or a list of strings")
+
+        requests: list[OmniDiffusionRequest] = []
+        for p in prompts:
+            requests.append(
+                prepare_requests(
+                    p,
+                    **kwargs,
+                )
+            )
+        logger.info(f"Prepared {len(requests)} requests for generation.")
+        return self._run_engine(requests)
+
+    def _run_engine(self, requests: list[OmniDiffusionRequest]):
+        return self.engine.step(requests)

vllm_omni/diffusion/registry.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import importlib
+
+from vllm_omni.diffusion.data import OmniDiffusionConfig
+
+_DIFFUSION_MODELS = {
+    # arch:(mod_folder, mod_relname, cls_name)
+    "QwenImagePipeline": (
+        "qwen_image",
+        "pipeline_qwen_image",
+        "QwenImagePipeline",
+    ),
+}
+
+
+def initialize_model(
+    od_config: OmniDiffusionConfig,
+):
+    if od_config.model_class_name in _DIFFUSION_MODELS:
+        mod_folder, mod_relname, cls_name = _DIFFUSION_MODELS[od_config.model_class_name]
+        module_name = f"vllm_omni.diffusion.models.{mod_folder}.{mod_relname}"
+        module = importlib.import_module(module_name)
+        model_class = getattr(module, cls_name)
+        model = model_class(od_config=od_config, prefix=mod_relname)
+        return model
+    else:
+        raise ValueError(f"Model class {od_config.model_class_name} not found in diffusion model registry.")
+
+
+_DIFFUSION_POST_PROCESS_FUNCS = {
+    # arch: post_process_func
+    # `post_process_func` function must be placed in {mod_folder}/{mod_relname}.py,
+    # where mod_folder and mod_relname are  defined and mapped using `_DIFFUSION_MODELS` via the `arch` key
+    "QwenImagePipeline": "get_qwen_image_post_process_func",
+}
+
+
+def get_diffusion_post_process_func(od_config: OmniDiffusionConfig):
+    if od_config.model_class_name in _DIFFUSION_POST_PROCESS_FUNCS:
+        mod_folder, mod_relname, _ = _DIFFUSION_MODELS[od_config.model_class_name]
+        func_name = _DIFFUSION_POST_PROCESS_FUNCS[od_config.model_class_name]
+        module_name = f"vllm_omni.diffusion.models.{mod_folder}.{mod_relname}"
+        module = importlib.import_module(module_name)
+        post_process_func = getattr(module, func_name)
+        return post_process_func(od_config)
+    else:
+        raise ValueError(
+            f"Post process function for model class {od_config.model_class_name} not found in diffusion model registry."
+        )

vllm_omni/diffusion/request.py

@@ -0,0 +1,179 @@
+# adapted from sglang and fastvideo
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pprint
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+import PIL.Image
+import torch
+
+
+@dataclass
+class OmniDiffusionRequest:
+    """
+    Complete state passed through the pipeline execution.
+
+    This dataclass contains all information needed during the diffusion pipeline
+    execution, allowing methods to update specific components without needing
+    to manage numerous individual parameters.
+    """
+
+    # TODO(will): double check that args are separate from server_args
+    # properly. Also maybe think about providing an abstraction for pipeline
+    # specific arguments.
+    # data_type: DataType
+
+    request_id: str | None = None
+
+    generator: torch.Generator | list[torch.Generator] | None = None
+
+    # Image inputs
+    image_path: str | None = None
+    # Image encoder hidden states
+    image_embeds: list[torch.Tensor] = field(default_factory=list)
+    pil_image: torch.Tensor | PIL.Image.Image | None = None
+    pixel_values: torch.Tensor | PIL.Image.Image | None = None
+    preprocessed_image: torch.Tensor | None = None
+
+    # Text inputs
+    prompt: str | list[str] | None = None
+    negative_prompt: str | list[str] | None = None
+    prompt_path: str | None = None
+    output_path: str = "outputs/"
+    # without extension
+    output_file_name: str | None = None
+    output_file_ext: str | None = None
+    # Primary encoder embeddings
+    prompt_embeds: list[torch.Tensor] | torch.Tensor = field(default_factory=list)
+    negative_prompt_embeds: list[torch.Tensor] | None = None
+    prompt_attention_mask: list[torch.Tensor] | None = None
+    negative_attention_mask: list[torch.Tensor] | None = None
+    clip_embedding_pos: list[torch.Tensor] | None = None
+    clip_embedding_neg: list[torch.Tensor] | None = None
+
+    pooled_embeds: list[torch.Tensor] = field(default_factory=list)
+    neg_pooled_embeds: list[torch.Tensor] = field(default_factory=list)
+
+    # Additional text-related parameters
+    max_sequence_length: int | None = None
+    prompt_template: dict[str, Any] | None = None
+    do_classifier_free_guidance: bool = False
+
+    # Batch info
+    num_outputs_per_prompt: int = 1
+    seed: int | None = None
+    seeds: list[int] | None = None
+
+    # Tracking if embeddings are already processed
+    is_prompt_processed: bool = False
+
+    # Latent tensors
+    latents: torch.Tensor | None = None
+    raw_latent_shape: torch.Tensor | None = None
+    noise_pred: torch.Tensor | None = None
+    image_latent: torch.Tensor | None = None
+
+    # Latent dimensions
+    height_latents: list[int] | int | None = None
+    width_latents: list[int] | int | None = None
+    num_frames: list[int] | int = 1  # Default for image models
+    num_frames_round_down: bool = False  # Whether to round down num_frames if it's not divisible by num_gpus
+
+    # Original dimensions (before VAE scaling)
+    height: list[int] | int | None = None
+    width: list[int] | int | None = None
+    fps: list[int] | int | None = None
+    height_not_provided: bool = False
+    width_not_provided: bool = False
+
+    # Timesteps
+    timesteps: torch.Tensor | None = None
+    timestep: torch.Tensor | float | int | None = None
+    step_index: int | None = None
+    boundary_ratio: float | None = None
+
+    # Scheduler parameters
+    num_inference_steps: int = 50
+    guidance_scale: float = 1.0
+    guidance_scale_2: float | None = None
+    guidance_rescale: float = 0.0
+    eta: float = 0.0
+    sigmas: list[float] | None = None
+
+    true_cfg_scale: float | None = None  # qwen-image specific now
+
+    n_tokens: int | None = None
+
+    # Other parameters that may be needed by specific schedulers
+    extra_step_kwargs: dict[str, Any] = field(default_factory=dict)
+
+    # Component modules (populated by the pipeline)
+    modules: dict[str, Any] = field(default_factory=dict)
+
+    return_trajectory_latents: bool = False
+    return_trajectory_decoded: bool = False
+    trajectory_timesteps: list[torch.Tensor] | None = None
+    trajectory_latents: torch.Tensor | None = None
+
+    # Extra parameters that might be needed by specific pipeline implementations
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    # Misc
+    save_output: bool = True
+    return_frames: bool = False
+
+    # TeaCache parameters
+    enable_teacache: bool = False
+    # teacache_params: TeaCacheParams | WanTeaCacheParams | None = None
+
+    # STA parameters
+    STA_param: list | None = None
+    is_cfg_negative: bool = False
+    mask_search_final_result_pos: list[list] | None = None
+    mask_search_final_result_neg: list[list] | None = None
+
+    # VSA parameters
+    VSA_sparsity: float = 0.0
+    # perf_logger: PerformanceLogger | None = None
+
+    # stage logging
+    # logging_info: PipelineLoggingInfo = field(default_factory=PipelineLoggingInfo)
+
+    # profile
+    profile: bool = False
+    num_profiled_timesteps: int = 8
+
+    # debugging
+    debug: bool = False
+
+    # results
+    output: torch.Tensor | None = None
+
+    @property
+    def batch_size(self):
+        # Determine batch size
+        if isinstance(self.prompt, list):
+            batch_size = len(self.prompt)
+        elif self.prompt is not None:
+            batch_size = 1
+        else:
+            batch_size = self.prompt_embeds[0].shape[0]
+
+        # Adjust batch size for number of videos per prompt
+        batch_size *= self.num_outputs_per_prompt
+        return batch_size
+
+    def __post_init__(self):
+        """Initialize dependent fields after dataclass initialization."""
+        # Set do_classifier_free_guidance based on guidance scale and negative prompt
+        if self.guidance_scale > 1.0 and self.negative_prompt is not None:
+            self.do_classifier_free_guidance = True
+        if self.negative_prompt_embeds is None:
+            self.negative_prompt_embeds = []
+        if self.guidance_scale_2 is None:
+            self.guidance_scale_2 = self.guidance_scale
+
+    def __str__(self):
+        return pprint.pformat(asdict(self), indent=2, width=120)

vllm_omni/diffusion/utils/hf_utils.py

@@ -0,0 +1,20 @@
+from functools import lru_cache
+
+from diffusers.pipelines.pipeline_utils import DiffusionPipeline
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+
+def load_diffusers_config(model_name) -> dict:
+    config = DiffusionPipeline.load_config(model_name)
+    return config
+
+
+@lru_cache
+def is_diffusion_model(model_name: str) -> bool:
+    try:
+        load_diffusers_config(model_name)
+        return True
+    except Exception:
+        return False

vllm_omni/diffusion/utils/network_utils.py

@@ -0,0 +1,19 @@
+import socket
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+
+def is_port_available(port):
+    """Return whether a port is available."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        try:
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            s.bind(("", port))
+            s.listen(1)
+            return True
+        except OSError:
+            return False
+        except OverflowError:
+            return False

vllm_omni/diffusion/worker/gpu_worker.py

@@ -0,0 +1,196 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import multiprocessing as mp
+import os
+
+import torch
+import zmq
+from vllm.config import VllmConfig, set_current_vllm_config
+from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
+from vllm.distributed.parallel_state import (
+    init_distributed_environment,
+    initialize_model_parallel,
+)
+from vllm.logger import init_logger
+
+from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
+from vllm_omni.diffusion.registry import initialize_model
+from vllm_omni.diffusion.request import OmniDiffusionRequest
+
+logger = init_logger(__name__)
+
+
+class GPUWorker:
+    """
+    A worker that executes the model on a single GPU.
+    """
+
+    def __init__(
+        self,
+        local_rank: int,
+        rank: int,
+        od_config: OmniDiffusionConfig,
+    ):
+        self.local_rank = local_rank
+        self.rank = rank
+        self.od_config = od_config
+        self.pipeline = None
+
+        self.init_device_and_model()
+
+    def init_device_and_model(self) -> None:
+        """Initialize the device and load the model."""
+        world_size = self.od_config.num_gpus
+        rank = self.rank
+        # Set environment variables for distributed initialization
+        os.environ["MASTER_ADDR"] = "localhost"
+        os.environ["MASTER_PORT"] = str(self.od_config.master_port)
+        os.environ["LOCAL_RANK"] = str(self.local_rank)
+        os.environ["RANK"] = str(rank)
+        os.environ["WORLD_SIZE"] = str(world_size)
+
+        device = torch.device(f"cuda:{rank}")
+        torch.cuda.set_device(device)
+
+        # hack
+        vllm_config = VllmConfig()
+        vllm_config.parallel_config.tensor_parallel_size = self.od_config.num_gpus
+        set_current_vllm_config(vllm_config)
+
+        init_distributed_environment(world_size=world_size, rank=rank)
+        initialize_model_parallel(tensor_model_parallel_size=world_size)
+
+        with device:
+            self.pipeline = initialize_model(self.od_config)
+            self.pipeline.load_weights()
+            self.pipeline.eval()
+        logger.info(f"Worker {self.rank}: Initialized device, model, and distributed environment.")
+        logger.info(f"Worker {self.rank}: Model loaded successfully.")
+
+    @torch.inference_mode()
+    def execute_model(self, reqs: list[OmniDiffusionRequest], od_config: OmniDiffusionConfig) -> DiffusionOutput:
+        """
+        Execute a forward pass.
+        """
+        assert self.pipeline is not None
+        # TODO: dealing with first req for now
+        req = reqs[0]
+        output = self.pipeline.forward(req)
+        return output
+
+
+class WorkerProc:
+    """Wrapper that runs one Worker in a separate process."""
+
+    def __init__(
+        self,
+        od_config: OmniDiffusionConfig,
+        gpu_id: int,
+        broadcast_handle,
+    ):
+        self.od_config = od_config
+
+        # Inter-process Communication
+        self.context = zmq.Context(io_threads=2)
+
+        # Initialize MessageQueue reader from handle
+        self.mq = MessageQueue.create_from_handle(broadcast_handle, gpu_id)
+
+        self.result_mq = None
+        self.result_mq_handle = None
+
+        # Setup result sender (only for rank 0 for now, or whoever needs to reply)
+        # Assuming only rank 0 replies to scheduler as per original logic
+        if gpu_id == 0:
+            # Create MessageQueue for results (1 writer -> 1 reader)
+            # We assume the reader (SyncScheduler) will act as rank 0
+            self.result_mq = MessageQueue(n_reader=1, n_local_reader=1, local_reader_ranks=[0])
+            self.result_mq_handle = self.result_mq.export_handle()
+            logger.info(f"Worker {gpu_id} created result MessageQueue")
+
+        assert od_config.master_port is not None
+        worker = GPUWorker(
+            local_rank=gpu_id,
+            rank=gpu_id,
+            od_config=od_config,
+        )
+        self.worker = worker
+        self.gpu_id = gpu_id
+        self._running = True
+
+    def return_result(self, output: DiffusionOutput):
+        """
+        replies to client, only on rank 0
+        """
+        if self.result_mq is not None:
+            self.result_mq.enqueue(output)
+
+    def recv_reqs(self):
+        """
+        Receive requests from broadcast queue
+        """
+        return self.mq.dequeue()
+
+    # TODO: queueing, cancellation
+    def worker_busy_loop(self) -> None:
+        """Main busy loop for Multiprocessing Workers"""
+
+        logger.info(f"Worker {self.gpu_id} ready to receive requests via shared memory")
+
+        while self._running:
+            reqs = None
+            # 1: receive requests
+            try:
+                reqs = self.recv_reqs()
+            except Exception as e:
+                logger.error(
+                    f"Error receiving requests in scheduler event loop: {e}",
+                    exc_info=True,
+                )
+                continue
+
+            # 2: execute, make sure a reply is always sent
+            try:
+                output = self.worker.execute_model(reqs, self.od_config)
+            except Exception as e:
+                logger.error(
+                    f"Error executing forward in event loop: {e}",
+                    exc_info=True,
+                )
+                output = DiffusionOutput(error=str(e))
+
+            try:
+                self.return_result(output)
+            except zmq.ZMQError as e:
+                # Reply failed; log and keep loop alive to accept future requests
+                logger.error(f"ZMQ error sending reply: {e}")
+                continue
+
+        logger.info("event loop terminated.")
+        # if self.result_sender is not None:
+        #     self.result_sender.close()
+        self.context.term()
+
+    @staticmethod
+    def worker_main(
+        rank: int,
+        od_config: OmniDiffusionConfig,
+        pipe_writer: mp.connection.Connection,
+        broadcast_handle,
+    ) -> None:
+        """Worker initialization and execution loops."""
+
+        worker_proc = WorkerProc(
+            od_config,
+            gpu_id=rank,
+            broadcast_handle=broadcast_handle,
+        )
+        logger.info(f"Worker {rank}: Scheduler loop started.")
+        pipe_writer.send(
+            {
+                "status": "ready",
+                "result_handle": worker_proc.result_mq_handle if rank == 0 else None,
+            }
+        )
+        worker_proc.worker_busy_loop()
+        logger.info(f"Worker {rank}: Shutdown complete.")

vllm_omni/entrypoints/omni.py

@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# unify OmniLLM and OmniDiffusion entrypoints
+
+from vllm_omni.diffusion.omni_diffusion import OmniDiffusion
+from vllm_omni.diffusion.utils.hf_utils import is_diffusion_model
+from vllm_omni.entrypoints.omni_llm import OmniLLM
+
+
+class Omni:
+    def __init__(self, *args, **kwargs):
+        model = args[0] if args else kwargs.get("model", "")
+        if is_diffusion_model(model):
+            self.instance: OmniLLM | OmniDiffusion = OmniDiffusion(*args, **kwargs)
+        else:
+            self.instance: OmniLLM | OmniDiffusion = OmniLLM(*args, **kwargs)
+
+    def __getattr__(self, name):
+        """Delegate attribute access to the chosen backend instance."""
+        return getattr(self.instance, name)
+
+    def generate(self, *args, **kwargs):
+        """Convenience wrapper to call `generate` on the backend if available."""
+        if hasattr(self.instance, "generate"):
+            return getattr(self.instance, "generate")(*args, **kwargs)
+        raise AttributeError(f"'{self.instance.__class__.__name__}' has no attribute 'generate'")

vllm_omni/entrypoints/stage_utils.py

@@ -84,7 +84,7 @@ def set_stage_gpu_devices(stage_id: int, devices: str | int | None) -> None:
             )
 
         try:
-            import torch  # noqa: WPS433
+            import torch
 
             if torch.cuda.is_available():
                 try: