[Core][Diffusion] Refactor prompt updates into a modality-ready interaction framework (#6294)
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
1 parent
eeee257913
commit
2df95de421
18 files changed
+1079
-362
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
# Adapted from Helios (https://github.com/BestWishYsh/Helios)
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -18,18 +18,20 @@ from diffusers import AutoencoderKLWan
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
from torch import nn
|
||||
from transformers import AutoConfig, AutoTokenizer, UMT5EncoderModel
|
||||
from typing_extensions import override
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.interaction.mixin import InteractionMixin
|
||||
from vllm_omni.diffusion.interaction.types import ChunkMediaSpec
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.helios.helios_transformer import HeliosTransformer3DModel
|
||||
from vllm_omni.diffusion.models.helios.scheduling_helios import HeliosScheduler
|
||||
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
|
||||
from vllm_omni.diffusion.prompt_update import PromptUpdateMixin
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
@@ -160,7 +162,7 @@ class HeliosPipeline(
|
||||
CFGParallelMixin,
|
||||
ProgressBarMixin,
|
||||
DiffusionPipelineProfilerMixin,
|
||||
PromptUpdateMixin,
|
||||
InteractionMixin,
|
||||
SupportsComponentDiscovery,
|
||||
):
|
||||
"""Helios text-to-video / image-to-video / video-to-video pipeline for vllm-omni.
|
||||
@@ -533,10 +535,22 @@ class HeliosPipeline(
|
||||
"zero_steps": int(extra.get("zero_steps", 1)),
|
||||
}
|
||||
)
|
||||
self._prepare_next_chunk(state)
|
||||
self.prepare_next_chunk(state)
|
||||
return state
|
||||
|
||||
def _prepare_next_chunk(self, state: StepRequestState) -> None:
|
||||
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
|
||||
"""Expose this chunk's decoded media extent for interaction timelines."""
|
||||
num_frames = int(state.extra["window_num_frames"])
|
||||
fps = state.sampling.fps
|
||||
if fps is None or float(fps) <= 0:
|
||||
raise ValueError(
|
||||
"sampling.fps is required and must be > 0 for interaction modalities that use the "
|
||||
f"chunk media timeline, got {fps!r} (request_id={state.request_id!r})"
|
||||
)
|
||||
return ChunkMediaSpec(num_frames=num_frames, fps=float(fps))
|
||||
|
||||
@override
|
||||
def prepare_next_chunk(self, state: StepRequestState) -> None:
|
||||
extra = state.extra
|
||||
k = state.chunk_index
|
||||
is_first_chunk = k == 0
|
||||
@@ -900,14 +914,9 @@ class HeliosPipeline(
|
||||
|
||||
output = current_latents if extra["output_type"] == "latent" else current_video
|
||||
completed_chunk_index = state.chunk_index
|
||||
prompt_update_metadata = state.extra.pop("prompt_update_chunk_metadata", {})
|
||||
state.chunk_index += 1
|
||||
finished = state.request_denoise_completed
|
||||
if not finished:
|
||||
# Apply queued/advancing prompt updates before preparing the next chunk.
|
||||
self._apply_prompt_update_at_chunk_boundary(state)
|
||||
self._prepare_next_chunk(state)
|
||||
else:
|
||||
if finished:
|
||||
self._current_timestep = None
|
||||
if current_omni_platform.is_available():
|
||||
current_omni_platform.empty_cache()
|
||||
@@ -917,9 +926,6 @@ class HeliosPipeline(
|
||||
stage_durations=self.stage_durations if hasattr(self, "stage_durations") else {},
|
||||
chunk_index=completed_chunk_index,
|
||||
total_chunks=state.total_chunks,
|
||||
started_event_ids=prompt_update_metadata.get("started_event_ids", []),
|
||||
active_event_ids=prompt_update_metadata.get("active_event_ids", []),
|
||||
completed_event_ids=prompt_update_metadata.get("completed_event_ids", []),
|
||||
finished=finished,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
@@ -19,6 +19,7 @@ if TYPE_CHECKING:
|
||||
|
||||
from vllm_omni.diffusion.cache.cachedit import CacheDiTBackend
|
||||
from vllm_omni.diffusion.data import DiffusionOutput
|
||||
from vllm_omni.diffusion.interaction.types import ChunkMediaSpec
|
||||
from vllm_omni.diffusion.worker.input_batch import InputBatch
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
@@ -108,30 +109,34 @@ def supports_step_execution(pipeline: object) -> bool:
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsPromptUpdate(Protocol):
|
||||
"""Optional protocol for pipelines that support midway prompt updates.
|
||||
class SupportsInteractionApply(Protocol):
|
||||
"""Optional protocol for pipelines with unified mid-generation, chunk-boundary hooks."""
|
||||
|
||||
Pipelines typically implement this via
|
||||
:class:`~vllm_omni.diffusion.prompt_update.PromptUpdateMixin`.
|
||||
"""
|
||||
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
|
||||
"""Return the media timeline represented by the upcoming/current chunk.
|
||||
|
||||
supports_prompt_update: ClassVar[bool] = True
|
||||
Useful when interaction handler needs interpolation/integration on a frame-by-frame basis,
|
||||
or for backpressure/pacing.
|
||||
"""
|
||||
...
|
||||
|
||||
def prepare_prompt_update(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
prompt: str,
|
||||
event_id: str,
|
||||
transition_chunks: int | None = None,
|
||||
) -> None:
|
||||
"""Encode and queue a prompt update on request-local state."""
|
||||
def apply_interaction_at_chunk_boundary(self, state: StepRequestState) -> None:
|
||||
"""Advance queued interactions before the next generation chunk."""
|
||||
...
|
||||
|
||||
def prepare_next_chunk(self, state: StepRequestState) -> None:
|
||||
"""Set up pipeline state for the next chunk after interaction apply.
|
||||
|
||||
Default implementations is a no-op; model-specific pipelines override
|
||||
when chunk transitions require latent/history bookkeeping.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def supports_prompt_update(pipeline: object) -> bool:
|
||||
"""Return whether ``pipeline`` implements :class:`SupportsPromptUpdate`."""
|
||||
def supports_interaction_apply(pipeline: object) -> bool:
|
||||
"""Return whether ``pipeline`` implements :class:`SupportsInteractionApply`."""
|
||||
|
||||
return isinstance(pipeline, SupportsPromptUpdate)
|
||||
return isinstance(pipeline, SupportsInteractionApply)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
|
||||
Reference in new issue
Block a user