[Model][Perf] Optimize LTX Ulysses sequence parallelism (#7079)
Signed-off-by: mglyn <1203789601@qq.com>
This commit is contained in:
1 parent
f080f41e03
commit
3204a0b2ad
15 files changed
+685
-108
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Shared denoise execution primitives for LTX pipelines."""
|
||||
|
||||
@@ -578,12 +578,6 @@ class LTXPhaseExecutor:
|
||||
video_audio_step_adapter=video_audio_step_adapter,
|
||||
)
|
||||
video_coords, audio_coords = prepare_rope_coords_stage(pipeline, forward_ctx, latents, audio_latents)
|
||||
ring_degree = getattr(pipeline.od_config.parallel_config, "ring_degree", 1) or 1
|
||||
if padded_audio_num_frames > original_audio_num_frames and ring_degree > 1:
|
||||
raise ValueError(
|
||||
"LTX audio padding requires an attention mask, which Ring sequence parallelism does not support. "
|
||||
"Use Ulysses-only SP or choose a request whose audio latent length is divisible by the SP size."
|
||||
)
|
||||
denoise_ctx = LTXDenoiseContext(
|
||||
latents=latents,
|
||||
audio_latents=audio_latents,
|
||||
|
||||
@@ -497,7 +497,6 @@ class LTXRequestMixin:
|
||||
image_crf = _normalize_image_crf(first_image_crf if first_image_crf is not None else image_crf)
|
||||
|
||||
sampling = sampling_params_list[0]
|
||||
is_dummy_run = req.is_dummy_run()
|
||||
prompt = [item if isinstance(item, str) else (item.get("prompt") or "") for item in req.prompts] or prompt
|
||||
negative_prompt = _resolve_negative_prompts(
|
||||
req.prompts,
|
||||
@@ -511,8 +510,6 @@ class LTXRequestMixin:
|
||||
num_inference_steps = (
|
||||
sampling.num_inference_steps or num_inference_steps or self.pipeline_recipe.num_inference_steps
|
||||
)
|
||||
if is_dummy_run and self.pipeline_recipe.fixed_num_inference_steps:
|
||||
num_inference_steps = self.pipeline_recipe.num_inference_steps
|
||||
num_inference_steps = max(int(num_inference_steps), 2)
|
||||
|
||||
num_videos_per_prompt = (
|
||||
@@ -528,8 +525,6 @@ class LTXRequestMixin:
|
||||
for item in sampling_params_list
|
||||
]
|
||||
guidance = guidance_specs[0]
|
||||
if is_dummy_run and not self.pipeline_recipe.phases[0].allow_guidance_override:
|
||||
guidance = self.pipeline_recipe.request_guidance
|
||||
if any(item != guidance for item in guidance_specs[1:]):
|
||||
raise ValueError("Batched LTX requests must use identical video/audio guidance parameters.")
|
||||
|
||||
|
||||
@@ -5,14 +5,16 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from contextlib import nullcontext
|
||||
import time
|
||||
from collections.abc import Iterable, Iterator
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import replace
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import torch
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
from torch import nn
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
@@ -60,6 +62,8 @@ from .ltx2_request import (
|
||||
validate_pipeline_request,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _run_ltx_vocoder(vocoder: nn.Module, generated_mel: torch.Tensor) -> torch.Tensor:
|
||||
"""Run the BWE vocoder in FP32, matching the official LTX pipeline."""
|
||||
@@ -195,7 +199,7 @@ class LTXRuntime(
|
||||
connector_batches_cfg = False
|
||||
distributed_video_decode = True
|
||||
support_image_input = False
|
||||
dummy_run_num_frames = 1
|
||||
dummy_run_num_frames: ClassVar[int] = 0
|
||||
preserve_sp_padded_audio_duration = False
|
||||
reports_stage_durations = False
|
||||
|
||||
@@ -207,6 +211,14 @@ class LTXRuntime(
|
||||
f"{self.__class__.__name__} does not support ulysses_mode='advanced_uaa'. "
|
||||
"Use the default ulysses_mode='strict' for LTX sequence parallelism."
|
||||
)
|
||||
ring_degree = getattr(parallel_config, "ring_degree", 1)
|
||||
allgather_degree = getattr(parallel_config, "allgather_degree", 1)
|
||||
if ring_degree != 1 or allgather_degree != 1:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} supports pure Ulysses sequence parallelism only. "
|
||||
"Set ring_degree=1 and allgather_degree=1; "
|
||||
f"got {ring_degree=} and {allgather_degree=}."
|
||||
)
|
||||
self.model_version = detect_ltx_model_version(od_config.model, revision=getattr(od_config, "revision", None))
|
||||
self.use_diffusion_decoder = _ltx2_use_diffusion_decoder(od_config, self.model_version)
|
||||
self.component_profile = resolve_ltx_component_profile(self.pipeline_kind, self.model_version)
|
||||
@@ -354,27 +366,29 @@ class LTXRuntime(
|
||||
)
|
||||
)
|
||||
self._enter_phase(phase_recipe)
|
||||
phase_inputs = self._build_phase_inputs(
|
||||
request_inputs,
|
||||
phase_recipe,
|
||||
phase_results[-1] if phase_results else None,
|
||||
)
|
||||
with self._profile_phase_method("_build_phase_inputs", phase_recipe.name):
|
||||
phase_inputs = self._build_phase_inputs(
|
||||
request_inputs,
|
||||
phase_recipe,
|
||||
phase_results[-1] if phase_results else None,
|
||||
)
|
||||
if phase_sigmas is not None:
|
||||
phase_inputs = replace(phase_inputs, num_inference_steps=len(phase_sigmas) - 1)
|
||||
noise_scale = phase_recipe.noise_scale
|
||||
if override_sigmas is not None and phase_recipe.input_transform == "spatial_upsample":
|
||||
noise_scale = float(override_sigmas[0])
|
||||
phase_result = self.run_phase(
|
||||
req,
|
||||
phase_inputs,
|
||||
noise_scale=noise_scale,
|
||||
sigmas=phase_sigmas,
|
||||
timesteps=None,
|
||||
attention_kwargs=None,
|
||||
phase_recipe=phase_recipe,
|
||||
image=image,
|
||||
prompt_context=prompt_context,
|
||||
)
|
||||
with self._profile_phase_method("run_phase", phase_recipe.name):
|
||||
phase_result = self.run_phase(
|
||||
req,
|
||||
phase_inputs,
|
||||
noise_scale=noise_scale,
|
||||
sigmas=phase_sigmas,
|
||||
timesteps=None,
|
||||
attention_kwargs=None,
|
||||
phase_recipe=phase_recipe,
|
||||
image=image,
|
||||
prompt_context=prompt_context,
|
||||
)
|
||||
phase_results.append(phase_result)
|
||||
prompt_context = phase_result.forward_context.prompt_context
|
||||
|
||||
@@ -384,7 +398,45 @@ class LTXRuntime(
|
||||
video=phase_results[self.pipeline_recipe.video_output_phase].video,
|
||||
audio=phase_results[self.pipeline_recipe.audio_output_phase].audio,
|
||||
)
|
||||
return self.decode_phase(output_phase)
|
||||
with self._profile_phase_method("decode_phase"):
|
||||
output = self.decode_phase(output_phase)
|
||||
return self._attach_profiled_stage_durations(output)
|
||||
|
||||
@contextmanager
|
||||
def _profile_phase_method(self, method_name: str, phase_name: str | None = None) -> Iterator[None]:
|
||||
if not getattr(self, "enable_diffusion_pipeline_profiler", False):
|
||||
yield
|
||||
return
|
||||
|
||||
metric_parts = [self.__class__.__name__]
|
||||
if phase_name is not None:
|
||||
metric_parts.append(phase_name)
|
||||
metric_parts.append(method_name)
|
||||
metric_name = ".".join(metric_parts)
|
||||
|
||||
if current_omni_platform.is_available():
|
||||
current_omni_platform.synchronize()
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if current_omni_platform.is_available():
|
||||
current_omni_platform.synchronize()
|
||||
duration = time.perf_counter() - start_time
|
||||
logger.info("[DiffusionPipelineProfiler] %s took %.6fs", metric_name, duration)
|
||||
with self._profiler_lock:
|
||||
self._stage_durations[metric_name] = self._stage_durations.get(metric_name, 0.0) + duration
|
||||
|
||||
def _attach_profiled_stage_durations(
|
||||
self, output: DiffusionOutput | list[DiffusionOutput]
|
||||
) -> DiffusionOutput | list[DiffusionOutput]:
|
||||
if not getattr(self, "enable_diffusion_pipeline_profiler", False):
|
||||
return output
|
||||
stage_durations = self.stage_durations
|
||||
outputs = output if isinstance(output, list) else [output]
|
||||
for item in outputs:
|
||||
item.stage_durations = dict(stage_durations)
|
||||
return output
|
||||
|
||||
def _enter_phase(self, phase: LTXPhaseRecipe) -> None:
|
||||
self._active_phase_name = phase.name
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.parallel.base import (
|
||||
ParallelAttentionContext,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.comm import SeqAllToAll4D
|
||||
from vllm_omni.diffusion.distributed.group_coordinator import (
|
||||
SequenceParallelGroupCoordinator,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _VideoToAudioContext(ParallelAttentionContext):
|
||||
ulysses_pg: dist.ProcessGroup
|
||||
use_sync: bool
|
||||
|
||||
|
||||
class LTX2VideoToAudioParallelAttention:
|
||||
"""LTX video-to-audio SP with replicated audio queries.
|
||||
|
||||
Video K/V arrive sequence-sharded, while the much shorter audio query is
|
||||
replicated. Redistribute only K/V across sequence/head dimensions, slice
|
||||
audio query heads locally, then gather output heads. This uses three
|
||||
collectives instead of standard Ulysses' four and keeps audio replicated.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sp_group: SequenceParallelGroupCoordinator,
|
||||
*,
|
||||
use_sync: bool = False,
|
||||
) -> None:
|
||||
if sp_group.ring_world_size != 1:
|
||||
raise NotImplementedError(
|
||||
"LTX video-to-audio sequence parallelism currently supports pure Ulysses only (ring_degree must be 1)."
|
||||
)
|
||||
self._sp_group = sp_group
|
||||
self._ulysses_pg = sp_group.ulysses_group
|
||||
self._use_sync = use_sync
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "ltx2_video_to_audio"
|
||||
|
||||
def pre_attention(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata | None,
|
||||
):
|
||||
if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:
|
||||
raise ValueError(
|
||||
"LTX video-to-audio SP expects Q/K/V in (B, S, H, D) layout, "
|
||||
f"got {query.shape=}, {key.shape=}, {value.shape=}."
|
||||
)
|
||||
if key.shape != value.shape:
|
||||
raise ValueError(
|
||||
"LTX video-to-audio SP requires matching K/V shapes, "
|
||||
f"got key={tuple(key.shape)} and value={tuple(value.shape)}."
|
||||
)
|
||||
|
||||
world_size = self._sp_group.ulysses_world_size
|
||||
rank = self._sp_group.ulysses_rank
|
||||
query_heads = int(query.shape[2])
|
||||
if query_heads % world_size != 0:
|
||||
raise ValueError(
|
||||
"LTX video-to-audio SP requires query heads divisible by "
|
||||
f"ulysses_degree, got heads={query_heads}, ulysses_degree={world_size}."
|
||||
)
|
||||
if int(key.shape[2]) % world_size != 0:
|
||||
raise ValueError(
|
||||
"LTX video-to-audio SP requires K/V heads divisible by "
|
||||
f"ulysses_degree, got heads={key.shape[2]}, ulysses_degree={world_size}."
|
||||
)
|
||||
|
||||
heads_per_rank = query_heads // world_size
|
||||
head_start = rank * heads_per_rank
|
||||
query = query[:, :, head_start : head_start + heads_per_rank, :].contiguous()
|
||||
key = SeqAllToAll4D.apply(self._ulysses_pg, key, 2, 1, self._use_sync)
|
||||
value = SeqAllToAll4D.apply(self._ulysses_pg, value, 2, 1, self._use_sync)
|
||||
|
||||
ctx = _VideoToAudioContext(
|
||||
name=self.name,
|
||||
ulysses_pg=self._ulysses_pg,
|
||||
use_sync=self._use_sync,
|
||||
)
|
||||
return query, key, value, attn_metadata, ctx
|
||||
|
||||
def post_attention(
|
||||
self,
|
||||
attn_output: torch.Tensor,
|
||||
ctx: ParallelAttentionContext | None,
|
||||
) -> torch.Tensor:
|
||||
if not isinstance(ctx, _VideoToAudioContext):
|
||||
raise TypeError(f"Unexpected LTX video-to-audio context: {type(ctx)!r}.")
|
||||
|
||||
# Gather along heads by moving H to the leading dimension required by
|
||||
# all_gather_into_tensor, then restore the backend's (B, S, H, D) layout.
|
||||
local_heads_first = attn_output.permute(2, 0, 1, 3).contiguous()
|
||||
world_size = dist.get_world_size(ctx.ulysses_pg)
|
||||
gathered = torch.empty(
|
||||
(local_heads_first.shape[0] * world_size, *local_heads_first.shape[1:]),
|
||||
dtype=attn_output.dtype,
|
||||
device=attn_output.device,
|
||||
)
|
||||
dist.all_gather_into_tensor(gathered, local_heads_first, group=ctx.ulysses_pg)
|
||||
if ctx.use_sync:
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
current_omni_platform.synchronize()
|
||||
return gathered.permute(1, 2, 0, 3).contiguous()
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
# Copyright 2025 The Lightricks team and The HuggingFace Team.
|
||||
# All rights reserved.
|
||||
#
|
||||
@@ -47,9 +50,12 @@ from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
|
||||
from vllm_omni.diffusion.distributed.hsdp_utils import is_transformer_block_module
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_sp_group
|
||||
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput
|
||||
from vllm_omni.diffusion.forward_context import get_forward_context, is_forward_context_available
|
||||
|
||||
from .ltx2_sequence_parallel import LTX2VideoToAudioParallelAttention
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_RMSNORM_INIT_PARAMS = inspect.signature(RMSNorm.__init__).parameters
|
||||
@@ -359,6 +365,34 @@ def to_ltx_padding_mask(attention_mask: torch.Tensor) -> torch.Tensor:
|
||||
class _LTX2ParallelAttention(Attention):
|
||||
"""Preserve LTX SP collectives while applying model-specific padding masks."""
|
||||
|
||||
_SP_MODES = frozenset({"local", "video_self", "video_to_audio"})
|
||||
|
||||
def __init__(self, *args, ltx_sp_mode: str = "local", **kwargs):
|
||||
if ltx_sp_mode not in self._SP_MODES:
|
||||
raise ValueError(
|
||||
f"Unsupported LTX sequence-parallel mode {ltx_sp_mode!r}; expected one of {sorted(self._SP_MODES)}."
|
||||
)
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ltx_sp_mode = ltx_sp_mode
|
||||
self._video_to_audio_strategy = None
|
||||
|
||||
def _get_active_parallel_strategy(self):
|
||||
base_strategy = super()._get_active_parallel_strategy()
|
||||
if self.ltx_sp_mode == "local" or not base_strategy.enabled:
|
||||
return self._no_parallel_strategy
|
||||
if self.ltx_sp_mode == "video_self":
|
||||
return base_strategy
|
||||
if base_strategy.name != "ulysses":
|
||||
raise NotImplementedError(
|
||||
f"LTX video-to-audio sequence parallelism requires pure Ulysses; got strategy={base_strategy.name!r}."
|
||||
)
|
||||
if self._video_to_audio_strategy is None:
|
||||
self._video_to_audio_strategy = LTX2VideoToAudioParallelAttention(
|
||||
get_sp_group(),
|
||||
use_sync=self.use_sync,
|
||||
)
|
||||
return self._video_to_audio_strategy
|
||||
|
||||
def _run_local_attention(self, query, key, value, attn_metadata):
|
||||
attention_mask = attn_metadata.attn_mask if attn_metadata is not None else None
|
||||
if attention_mask is not None:
|
||||
@@ -597,6 +631,7 @@ class LTX2Attention(torch.nn.Module):
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
prefix: str = "",
|
||||
disable_kv_quant: bool = False,
|
||||
sequence_parallel_mode: str = "local",
|
||||
):
|
||||
super().__init__()
|
||||
# LTX-2 uses "rms_norm_across_heads", LTX-2.3 uses "rms_norm" -- both
|
||||
@@ -721,6 +756,7 @@ class LTX2Attention(torch.nn.Module):
|
||||
causal=False,
|
||||
prefix=prefix,
|
||||
disable_kv_quant=disable_kv_quant,
|
||||
ltx_sp_mode=sequence_parallel_mode,
|
||||
)
|
||||
|
||||
# LTX-2.3: per-head gated attention
|
||||
@@ -876,6 +912,7 @@ class LTX2VideoTransformerBlock(nn.Module):
|
||||
apply_gated_attention=video_gated_attn,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn1" if prefix else "attn1",
|
||||
sequence_parallel_mode="video_self",
|
||||
)
|
||||
|
||||
self.audio_norm1 = _make_rms_norm(audio_dim, eps=eps, elementwise_affine=elementwise_affine)
|
||||
@@ -965,6 +1002,7 @@ class LTX2VideoTransformerBlock(nn.Module):
|
||||
apply_gated_attention=audio_gated_attn,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.video_to_audio_attn" if prefix else "video_to_audio_attn",
|
||||
sequence_parallel_mode="video_to_audio",
|
||||
)
|
||||
|
||||
# 4. Feedforward layers
|
||||
@@ -1568,38 +1606,25 @@ class LTX2VideoTransformer3DModel(nn.Module):
|
||||
|
||||
return {
|
||||
"": {
|
||||
# Shard video/audio latents across sequence
|
||||
# Keep only the long video stream sequence-sharded. Audio and
|
||||
# prompt streams remain replicated on every SP rank.
|
||||
"hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False),
|
||||
"audio_hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False),
|
||||
"keyframes_mask": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False),
|
||||
# Shard prompt embeds across sequence
|
||||
"encoder_hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False),
|
||||
"audio_encoder_hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False),
|
||||
# Keep per-token video/audio timestep embeddings aligned with
|
||||
# their corresponding latent sequence shards. Official LTX
|
||||
# multi-guidance supplies both tensors as (B, seq_len).
|
||||
# Keep per-token video timestep embeddings aligned with the
|
||||
# video latent shard. LTX guidance supplies a separate,
|
||||
# replicated audio_timestep under SP.
|
||||
"timestep": SequenceParallelInput(split_dim=1, expected_dims=2, split_output=False),
|
||||
"audio_timestep": SequenceParallelInput(split_dim=1, expected_dims=2, split_output=False),
|
||||
},
|
||||
"rope": {
|
||||
0: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
|
||||
1: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
|
||||
},
|
||||
"audio_rope": {
|
||||
0: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
|
||||
1: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
|
||||
},
|
||||
"cross_attn_rope": {
|
||||
0: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
|
||||
1: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
|
||||
},
|
||||
"cross_attn_audio_rope": {
|
||||
0: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
|
||||
1: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
|
||||
},
|
||||
# Gather outputs before returning
|
||||
# Video output is sharded; audio output is already replicated.
|
||||
"proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
|
||||
"audio_proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
|
||||
}
|
||||
|
||||
def __init__(
|
||||
|
||||
Reference in new issue
Block a user