[Model][Perf] Optimize LTX Ulysses sequence parallelism (#7079)

Signed-off-by: mglyn <1203789601@qq.com>
This commit is contained in:
Mu GuanLin authored and GitHub committed 2026-09-07 10:48:26 +00:00
1 parent f080f41e03
commit 3204a0b2ad
15 files changed
+685 -108

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shared denoise execution primitives for LTX pipelines."""
@@ -578,12 +578,6 @@ class LTXPhaseExecutor:
video_audio_step_adapter=video_audio_step_adapter,
)
video_coords, audio_coords = prepare_rope_coords_stage(pipeline, forward_ctx, latents, audio_latents)
ring_degree = getattr(pipeline.od_config.parallel_config, "ring_degree", 1) or 1
if padded_audio_num_frames > original_audio_num_frames and ring_degree > 1:
raise ValueError(
"LTX audio padding requires an attention mask, which Ring sequence parallelism does not support. "
"Use Ulysses-only SP or choose a request whose audio latent length is divisible by the SP size."
)
denoise_ctx = LTXDenoiseContext(
latents=latents,
audio_latents=audio_latents,
@@ -497,7 +497,6 @@ class LTXRequestMixin:
image_crf = _normalize_image_crf(first_image_crf if first_image_crf is not None else image_crf)
sampling = sampling_params_list[0]
is_dummy_run = req.is_dummy_run()
prompt = [item if isinstance(item, str) else (item.get("prompt") or "") for item in req.prompts] or prompt
negative_prompt = _resolve_negative_prompts(
req.prompts,
@@ -511,8 +510,6 @@ class LTXRequestMixin:
num_inference_steps = (
sampling.num_inference_steps or num_inference_steps or self.pipeline_recipe.num_inference_steps
)
if is_dummy_run and self.pipeline_recipe.fixed_num_inference_steps:
num_inference_steps = self.pipeline_recipe.num_inference_steps
num_inference_steps = max(int(num_inference_steps), 2)
num_videos_per_prompt = (
@@ -528,8 +525,6 @@ class LTXRequestMixin:
for item in sampling_params_list
]
guidance = guidance_specs[0]
if is_dummy_run and not self.pipeline_recipe.phases[0].allow_guidance_override:
guidance = self.pipeline_recipe.request_guidance
if any(item != guidance for item in guidance_specs[1:]):
raise ValueError("Batched LTX requests must use identical video/audio guidance parameters.")
+72 -20
View File
@@ -5,14 +5,16 @@
from __future__ import annotations
from collections.abc import Iterable
from contextlib import nullcontext
import time
from collections.abc import Iterable, Iterator
from contextlib import contextmanager, nullcontext
from dataclasses import replace
from typing import Any, ClassVar
import torch
from diffusers.utils.torch_utils import randn_tensor
from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
@@ -60,6 +62,8 @@ from .ltx2_request import (
validate_pipeline_request,
)
logger = init_logger(__name__)
def _run_ltx_vocoder(vocoder: nn.Module, generated_mel: torch.Tensor) -> torch.Tensor:
"""Run the BWE vocoder in FP32, matching the official LTX pipeline."""
@@ -195,7 +199,7 @@ class LTXRuntime(
connector_batches_cfg = False
distributed_video_decode = True
support_image_input = False
dummy_run_num_frames = 1
dummy_run_num_frames: ClassVar[int] = 0
preserve_sp_padded_audio_duration = False
reports_stage_durations = False
@@ -207,6 +211,14 @@ class LTXRuntime(
f"{self.__class__.__name__} does not support ulysses_mode='advanced_uaa'. "
"Use the default ulysses_mode='strict' for LTX sequence parallelism."
)
ring_degree = getattr(parallel_config, "ring_degree", 1)
allgather_degree = getattr(parallel_config, "allgather_degree", 1)
if ring_degree != 1 or allgather_degree != 1:
raise ValueError(
f"{self.__class__.__name__} supports pure Ulysses sequence parallelism only. "
"Set ring_degree=1 and allgather_degree=1; "
f"got {ring_degree=} and {allgather_degree=}."
)
self.model_version = detect_ltx_model_version(od_config.model, revision=getattr(od_config, "revision", None))
self.use_diffusion_decoder = _ltx2_use_diffusion_decoder(od_config, self.model_version)
self.component_profile = resolve_ltx_component_profile(self.pipeline_kind, self.model_version)
@@ -354,27 +366,29 @@ class LTXRuntime(
)
)
self._enter_phase(phase_recipe)
phase_inputs = self._build_phase_inputs(
request_inputs,
phase_recipe,
phase_results[-1] if phase_results else None,
)
with self._profile_phase_method("_build_phase_inputs", phase_recipe.name):
phase_inputs = self._build_phase_inputs(
request_inputs,
phase_recipe,
phase_results[-1] if phase_results else None,
)
if phase_sigmas is not None:
phase_inputs = replace(phase_inputs, num_inference_steps=len(phase_sigmas) - 1)
noise_scale = phase_recipe.noise_scale
if override_sigmas is not None and phase_recipe.input_transform == "spatial_upsample":
noise_scale = float(override_sigmas[0])
phase_result = self.run_phase(
req,
phase_inputs,
noise_scale=noise_scale,
sigmas=phase_sigmas,
timesteps=None,
attention_kwargs=None,
phase_recipe=phase_recipe,
image=image,
prompt_context=prompt_context,
)
with self._profile_phase_method("run_phase", phase_recipe.name):
phase_result = self.run_phase(
req,
phase_inputs,
noise_scale=noise_scale,
sigmas=phase_sigmas,
timesteps=None,
attention_kwargs=None,
phase_recipe=phase_recipe,
image=image,
prompt_context=prompt_context,
)
phase_results.append(phase_result)
prompt_context = phase_result.forward_context.prompt_context
@@ -384,7 +398,45 @@ class LTXRuntime(
video=phase_results[self.pipeline_recipe.video_output_phase].video,
audio=phase_results[self.pipeline_recipe.audio_output_phase].audio,
)
return self.decode_phase(output_phase)
with self._profile_phase_method("decode_phase"):
output = self.decode_phase(output_phase)
return self._attach_profiled_stage_durations(output)
@contextmanager
def _profile_phase_method(self, method_name: str, phase_name: str | None = None) -> Iterator[None]:
if not getattr(self, "enable_diffusion_pipeline_profiler", False):
yield
return
metric_parts = [self.__class__.__name__]
if phase_name is not None:
metric_parts.append(phase_name)
metric_parts.append(method_name)
metric_name = ".".join(metric_parts)
if current_omni_platform.is_available():
current_omni_platform.synchronize()
start_time = time.perf_counter()
try:
yield
finally:
if current_omni_platform.is_available():
current_omni_platform.synchronize()
duration = time.perf_counter() - start_time
logger.info("[DiffusionPipelineProfiler] %s took %.6fs", metric_name, duration)
with self._profiler_lock:
self._stage_durations[metric_name] = self._stage_durations.get(metric_name, 0.0) + duration
def _attach_profiled_stage_durations(
self, output: DiffusionOutput | list[DiffusionOutput]
) -> DiffusionOutput | list[DiffusionOutput]:
if not getattr(self, "enable_diffusion_pipeline_profiler", False):
return output
stage_durations = self.stage_durations
outputs = output if isinstance(output, list) else [output]
for item in outputs:
item.stage_durations = dict(stage_durations)
return output
def _enter_phase(self, phase: LTXPhaseRecipe) -> None:
self._active_phase_name = phase.name
@@ -0,0 +1,125 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
from dataclasses import dataclass
import torch
import torch.distributed as dist
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.parallel.base import (
ParallelAttentionContext,
)
from vllm_omni.diffusion.distributed.comm import SeqAllToAll4D
from vllm_omni.diffusion.distributed.group_coordinator import (
SequenceParallelGroupCoordinator,
)
@dataclass(frozen=True, slots=True)
class _VideoToAudioContext(ParallelAttentionContext):
ulysses_pg: dist.ProcessGroup
use_sync: bool
class LTX2VideoToAudioParallelAttention:
"""LTX video-to-audio SP with replicated audio queries.
Video K/V arrive sequence-sharded, while the much shorter audio query is
replicated. Redistribute only K/V across sequence/head dimensions, slice
audio query heads locally, then gather output heads. This uses three
collectives instead of standard Ulysses' four and keeps audio replicated.
"""
def __init__(
self,
sp_group: SequenceParallelGroupCoordinator,
*,
use_sync: bool = False,
) -> None:
if sp_group.ring_world_size != 1:
raise NotImplementedError(
"LTX video-to-audio sequence parallelism currently supports pure Ulysses only (ring_degree must be 1)."
)
self._sp_group = sp_group
self._ulysses_pg = sp_group.ulysses_group
self._use_sync = use_sync
@property
def enabled(self) -> bool:
return True
@property
def name(self) -> str:
return "ltx2_video_to_audio"
def pre_attention(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: AttentionMetadata | None,
):
if query.ndim != 4 or key.ndim != 4 or value.ndim != 4:
raise ValueError(
"LTX video-to-audio SP expects Q/K/V in (B, S, H, D) layout, "
f"got {query.shape=}, {key.shape=}, {value.shape=}."
)
if key.shape != value.shape:
raise ValueError(
"LTX video-to-audio SP requires matching K/V shapes, "
f"got key={tuple(key.shape)} and value={tuple(value.shape)}."
)
world_size = self._sp_group.ulysses_world_size
rank = self._sp_group.ulysses_rank
query_heads = int(query.shape[2])
if query_heads % world_size != 0:
raise ValueError(
"LTX video-to-audio SP requires query heads divisible by "
f"ulysses_degree, got heads={query_heads}, ulysses_degree={world_size}."
)
if int(key.shape[2]) % world_size != 0:
raise ValueError(
"LTX video-to-audio SP requires K/V heads divisible by "
f"ulysses_degree, got heads={key.shape[2]}, ulysses_degree={world_size}."
)
heads_per_rank = query_heads // world_size
head_start = rank * heads_per_rank
query = query[:, :, head_start : head_start + heads_per_rank, :].contiguous()
key = SeqAllToAll4D.apply(self._ulysses_pg, key, 2, 1, self._use_sync)
value = SeqAllToAll4D.apply(self._ulysses_pg, value, 2, 1, self._use_sync)
ctx = _VideoToAudioContext(
name=self.name,
ulysses_pg=self._ulysses_pg,
use_sync=self._use_sync,
)
return query, key, value, attn_metadata, ctx
def post_attention(
self,
attn_output: torch.Tensor,
ctx: ParallelAttentionContext | None,
) -> torch.Tensor:
if not isinstance(ctx, _VideoToAudioContext):
raise TypeError(f"Unexpected LTX video-to-audio context: {type(ctx)!r}.")
# Gather along heads by moving H to the leading dimension required by
# all_gather_into_tensor, then restore the backend's (B, S, H, D) layout.
local_heads_first = attn_output.permute(2, 0, 1, 3).contiguous()
world_size = dist.get_world_size(ctx.ulysses_pg)
gathered = torch.empty(
(local_heads_first.shape[0] * world_size, *local_heads_first.shape[1:]),
dtype=attn_output.dtype,
device=attn_output.device,
)
dist.all_gather_into_tensor(gathered, local_heads_first, group=ctx.ulysses_pg)
if ctx.use_sync:
from vllm_omni.platforms import current_omni_platform
current_omni_platform.synchronize()
return gathered.permute(1, 2, 0, 3).contiguous()
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# Copyright 2025 The Lightricks team and The HuggingFace Team.
# All rights reserved.
#
@@ -47,9 +50,12 @@ from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.cache.cachedit import CacheDiTAdapterConfig
from vllm_omni.diffusion.distributed.hsdp_utils import is_transformer_block_module
from vllm_omni.diffusion.distributed.parallel_state import get_sp_group
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput, SequenceParallelOutput
from vllm_omni.diffusion.forward_context import get_forward_context, is_forward_context_available
from .ltx2_sequence_parallel import LTX2VideoToAudioParallelAttention
logger = init_logger(__name__)
_RMSNORM_INIT_PARAMS = inspect.signature(RMSNorm.__init__).parameters
@@ -359,6 +365,34 @@ def to_ltx_padding_mask(attention_mask: torch.Tensor) -> torch.Tensor:
class _LTX2ParallelAttention(Attention):
"""Preserve LTX SP collectives while applying model-specific padding masks."""
_SP_MODES = frozenset({"local", "video_self", "video_to_audio"})
def __init__(self, *args, ltx_sp_mode: str = "local", **kwargs):
if ltx_sp_mode not in self._SP_MODES:
raise ValueError(
f"Unsupported LTX sequence-parallel mode {ltx_sp_mode!r}; expected one of {sorted(self._SP_MODES)}."
)
super().__init__(*args, **kwargs)
self.ltx_sp_mode = ltx_sp_mode
self._video_to_audio_strategy = None
def _get_active_parallel_strategy(self):
base_strategy = super()._get_active_parallel_strategy()
if self.ltx_sp_mode == "local" or not base_strategy.enabled:
return self._no_parallel_strategy
if self.ltx_sp_mode == "video_self":
return base_strategy
if base_strategy.name != "ulysses":
raise NotImplementedError(
f"LTX video-to-audio sequence parallelism requires pure Ulysses; got strategy={base_strategy.name!r}."
)
if self._video_to_audio_strategy is None:
self._video_to_audio_strategy = LTX2VideoToAudioParallelAttention(
get_sp_group(),
use_sync=self.use_sync,
)
return self._video_to_audio_strategy
def _run_local_attention(self, query, key, value, attn_metadata):
attention_mask = attn_metadata.attn_mask if attn_metadata is not None else None
if attention_mask is not None:
@@ -597,6 +631,7 @@ class LTX2Attention(torch.nn.Module):
quant_config: "QuantizationConfig | None" = None,
prefix: str = "",
disable_kv_quant: bool = False,
sequence_parallel_mode: str = "local",
):
super().__init__()
# LTX-2 uses "rms_norm_across_heads", LTX-2.3 uses "rms_norm" -- both
@@ -721,6 +756,7 @@ class LTX2Attention(torch.nn.Module):
causal=False,
prefix=prefix,
disable_kv_quant=disable_kv_quant,
ltx_sp_mode=sequence_parallel_mode,
)
# LTX-2.3: per-head gated attention
@@ -876,6 +912,7 @@ class LTX2VideoTransformerBlock(nn.Module):
apply_gated_attention=video_gated_attn,
quant_config=quant_config,
prefix=f"{prefix}.attn1" if prefix else "attn1",
sequence_parallel_mode="video_self",
)
self.audio_norm1 = _make_rms_norm(audio_dim, eps=eps, elementwise_affine=elementwise_affine)
@@ -965,6 +1002,7 @@ class LTX2VideoTransformerBlock(nn.Module):
apply_gated_attention=audio_gated_attn,
quant_config=quant_config,
prefix=f"{prefix}.video_to_audio_attn" if prefix else "video_to_audio_attn",
sequence_parallel_mode="video_to_audio",
)
# 4. Feedforward layers
@@ -1568,38 +1606,25 @@ class LTX2VideoTransformer3DModel(nn.Module):
return {
"": {
# Shard video/audio latents across sequence
# Keep only the long video stream sequence-sharded. Audio and
# prompt streams remain replicated on every SP rank.
"hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False),
"audio_hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False),
"keyframes_mask": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False),
# Shard prompt embeds across sequence
"encoder_hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False),
"audio_encoder_hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, split_output=False),
# Keep per-token video/audio timestep embeddings aligned with
# their corresponding latent sequence shards. Official LTX
# multi-guidance supplies both tensors as (B, seq_len).
# Keep per-token video timestep embeddings aligned with the
# video latent shard. LTX guidance supplies a separate,
# replicated audio_timestep under SP.
"timestep": SequenceParallelInput(split_dim=1, expected_dims=2, split_output=False),
"audio_timestep": SequenceParallelInput(split_dim=1, expected_dims=2, split_output=False),
},
"rope": {
0: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
1: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
},
"audio_rope": {
0: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
1: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
},
"cross_attn_rope": {
0: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
1: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
},
"cross_attn_audio_rope": {
0: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
1: SequenceParallelInput(split_dim=rope_split_dim, expected_dims=rope_expected_dims, split_output=True),
},
# Gather outputs before returning
# Video output is sharded; audio output is already replicated.
"proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
"audio_proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
}
def __init__(