# adapted from sglang and fastvideo # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pprint from dataclasses import asdict, dataclass, field from typing import Any import PIL.Image import torch @dataclass class OmniDiffusionRequest: """ Complete state passed through the pipeline execution. This dataclass contains all information needed during the diffusion pipeline execution, allowing methods to update specific components without needing to manage numerous individual parameters. """ # TODO(will): double check that args are separate from server_args # properly. Also maybe think about providing an abstraction for pipeline # specific arguments. # data_type: DataType request_id: str | None = None generator: torch.Generator | list[torch.Generator] | None = None # Image inputs image_path: str | None = None # Image encoder hidden states image_embeds: list[torch.Tensor] = field(default_factory=list) pil_image: torch.Tensor | PIL.Image.Image | None = None pixel_values: torch.Tensor | PIL.Image.Image | None = None preprocessed_image: torch.Tensor | None = None # Text inputs prompt: str | list[str] | None = None negative_prompt: str | list[str] | None = None prompt_path: str | None = None output_path: str = "outputs/" # without extension output_file_name: str | None = None output_file_ext: str | None = None # Primary encoder embeddings prompt_embeds: list[torch.Tensor] | torch.Tensor = field(default_factory=list) negative_prompt_embeds: list[torch.Tensor] | None = None prompt_attention_mask: list[torch.Tensor] | None = None negative_attention_mask: list[torch.Tensor] | None = None clip_embedding_pos: list[torch.Tensor] | None = None clip_embedding_neg: list[torch.Tensor] | None = None pooled_embeds: list[torch.Tensor] = field(default_factory=list) neg_pooled_embeds: list[torch.Tensor] = field(default_factory=list) # Additional text-related parameters max_sequence_length: int | None = None prompt_template: dict[str, Any] | None = None do_classifier_free_guidance: bool = False # Batch info num_outputs_per_prompt: int = 1 seed: int | None = None seeds: list[int] | None = None # Tracking if embeddings are already processed is_prompt_processed: bool = False # Latent tensors latents: torch.Tensor | None = None raw_latent_shape: torch.Tensor | None = None noise_pred: torch.Tensor | None = None image_latent: torch.Tensor | None = None # Latent dimensions height_latents: list[int] | int | None = None width_latents: list[int] | int | None = None num_frames: list[int] | int = 1 # Default for image models num_frames_round_down: bool = False # Whether to round down num_frames if it's not divisible by num_gpus # Original dimensions (before VAE scaling) height: list[int] | int | None = None width: list[int] | int | None = None fps: list[int] | int | None = None height_not_provided: bool = False width_not_provided: bool = False # Timesteps timesteps: torch.Tensor | None = None timestep: torch.Tensor | float | int | None = None step_index: int | None = None boundary_ratio: float | None = None # Scheduler parameters num_inference_steps: int = 50 guidance_scale: float = 1.0 guidance_scale_2: float | None = None guidance_rescale: float = 0.0 eta: float = 0.0 sigmas: list[float] | None = None true_cfg_scale: float | None = None # qwen-image specific now n_tokens: int | None = None # Other parameters that may be needed by specific schedulers extra_step_kwargs: dict[str, Any] = field(default_factory=dict) # Component modules (populated by the pipeline) modules: dict[str, Any] = field(default_factory=dict) return_trajectory_latents: bool = False return_trajectory_decoded: bool = False trajectory_timesteps: list[torch.Tensor] | None = None trajectory_latents: torch.Tensor | None = None # Extra parameters that might be needed by specific pipeline implementations extra: dict[str, Any] = field(default_factory=dict) # Misc save_output: bool = True return_frames: bool = False # TeaCache parameters enable_teacache: bool = False # teacache_params: TeaCacheParams | WanTeaCacheParams | None = None # STA parameters STA_param: list | None = None is_cfg_negative: bool = False mask_search_final_result_pos: list[list] | None = None mask_search_final_result_neg: list[list] | None = None # VSA parameters VSA_sparsity: float = 0.0 # perf_logger: PerformanceLogger | None = None # stage logging # logging_info: PipelineLoggingInfo = field(default_factory=PipelineLoggingInfo) # profile profile: bool = False num_profiled_timesteps: int = 8 # debugging debug: bool = False # results output: torch.Tensor | None = None @property def batch_size(self): # Determine batch size if isinstance(self.prompt, list): batch_size = len(self.prompt) elif self.prompt is not None: batch_size = 1 else: batch_size = self.prompt_embeds[0].shape[0] # Adjust batch size for number of videos per prompt batch_size *= self.num_outputs_per_prompt return batch_size def __post_init__(self): """Initialize dependent fields after dataclass initialization.""" # Set do_classifier_free_guidance based on guidance scale and negative prompt if self.guidance_scale > 1.0 and self.negative_prompt is not None: self.do_classifier_free_guidance = True if self.negative_prompt_embeds is None: self.negative_prompt_embeds = [] if self.guidance_scale_2 is None: self.guidance_scale_2 = self.guidance_scale def __str__(self): return pprint.pformat(asdict(self), indent=2, width=120)