[Core][Diffusion] Refactor prompt updates into a modality-ready interaction framework (#6294)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
Zeyu Huang | 黃澤宇 authored and GitHub committed 2026-09-07 11:40:04 +08:00
1 parent eeee257913
commit 2df95de421
18 files changed
+1079 -362

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
# Adapted from Helios (https://github.com/BestWishYsh/Helios)
from __future__ import annotations
@@ -18,18 +18,20 @@ from diffusers import AutoencoderKLWan
from diffusers.utils.torch_utils import randn_tensor
from torch import nn
from transformers import AutoConfig, AutoTokenizer, UMT5EncoderModel
from typing_extensions import override
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.interaction.mixin import InteractionMixin
from vllm_omni.diffusion.interaction.types import ChunkMediaSpec
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.helios.helios_transformer import HeliosTransformer3DModel
from vllm_omni.diffusion.models.helios.scheduling_helios import HeliosScheduler
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.prompt_update import PromptUpdateMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.platforms import current_omni_platform
@@ -160,7 +162,7 @@ class HeliosPipeline(
CFGParallelMixin,
ProgressBarMixin,
DiffusionPipelineProfilerMixin,
PromptUpdateMixin,
InteractionMixin,
SupportsComponentDiscovery,
):
"""Helios text-to-video / image-to-video / video-to-video pipeline for vllm-omni.
@@ -533,10 +535,22 @@ class HeliosPipeline(
"zero_steps": int(extra.get("zero_steps", 1)),
}
)
self._prepare_next_chunk(state)
self.prepare_next_chunk(state)
return state
def _prepare_next_chunk(self, state: StepRequestState) -> None:
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
"""Expose this chunk's decoded media extent for interaction timelines."""
num_frames = int(state.extra["window_num_frames"])
fps = state.sampling.fps
if fps is None or float(fps) <= 0:
raise ValueError(
"sampling.fps is required and must be > 0 for interaction modalities that use the "
f"chunk media timeline, got {fps!r} (request_id={state.request_id!r})"
)
return ChunkMediaSpec(num_frames=num_frames, fps=float(fps))
@override
def prepare_next_chunk(self, state: StepRequestState) -> None:
extra = state.extra
k = state.chunk_index
is_first_chunk = k == 0
@@ -900,14 +914,9 @@ class HeliosPipeline(
output = current_latents if extra["output_type"] == "latent" else current_video
completed_chunk_index = state.chunk_index
prompt_update_metadata = state.extra.pop("prompt_update_chunk_metadata", {})
state.chunk_index += 1
finished = state.request_denoise_completed
if not finished:
# Apply queued/advancing prompt updates before preparing the next chunk.
self._apply_prompt_update_at_chunk_boundary(state)
self._prepare_next_chunk(state)
else:
if finished:
self._current_timestep = None
if current_omni_platform.is_available():
current_omni_platform.empty_cache()
@@ -917,9 +926,6 @@ class HeliosPipeline(
stage_durations=self.stage_durations if hasattr(self, "stage_durations") else {},
chunk_index=completed_chunk_index,
total_chunks=state.total_chunks,
started_event_ids=prompt_update_metadata.get("started_event_ids", []),
active_event_ids=prompt_update_metadata.get("active_event_ids", []),
completed_event_ids=prompt_update_metadata.get("completed_event_ids", []),
finished=finished,
)
+23 -18
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
from dataclasses import dataclass
@@ -19,6 +19,7 @@ if TYPE_CHECKING:
from vllm_omni.diffusion.cache.cachedit import CacheDiTBackend
from vllm_omni.diffusion.data import DiffusionOutput
from vllm_omni.diffusion.interaction.types import ChunkMediaSpec
from vllm_omni.diffusion.worker.input_batch import InputBatch
from vllm_omni.diffusion.worker.utils import StepRequestState
@@ -108,30 +109,34 @@ def supports_step_execution(pipeline: object) -> bool:
@runtime_checkable
class SupportsPromptUpdate(Protocol):
"""Optional protocol for pipelines that support midway prompt updates.
class SupportsInteractionApply(Protocol):
"""Optional protocol for pipelines with unified mid-generation, chunk-boundary hooks."""
Pipelines typically implement this via
:class:`~vllm_omni.diffusion.prompt_update.PromptUpdateMixin`.
"""
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
"""Return the media timeline represented by the upcoming/current chunk.
supports_prompt_update: ClassVar[bool] = True
Useful when interaction handler needs interpolation/integration on a frame-by-frame basis,
or for backpressure/pacing.
"""
...
def prepare_prompt_update(
self,
state: StepRequestState,
prompt: str,
event_id: str,
transition_chunks: int | None = None,
) -> None:
"""Encode and queue a prompt update on request-local state."""
def apply_interaction_at_chunk_boundary(self, state: StepRequestState) -> None:
"""Advance queued interactions before the next generation chunk."""
...
def prepare_next_chunk(self, state: StepRequestState) -> None:
"""Set up pipeline state for the next chunk after interaction apply.
Default implementations is a no-op; model-specific pipelines override
when chunk transitions require latent/history bookkeeping.
"""
...
def supports_prompt_update(pipeline: object) -> bool:
"""Return whether ``pipeline`` implements :class:`SupportsPromptUpdate`."""
def supports_interaction_apply(pipeline: object) -> bool:
"""Return whether ``pipeline`` implements :class:`SupportsInteractionApply`."""
return isinstance(pipeline, SupportsPromptUpdate)
return isinstance(pipeline, SupportsInteractionApply)
@runtime_checkable