[Frontend] Add midway prompt update for streaming video generation (#4652)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
Zeyu Huang | 黃澤宇 authored and GitHub committed 2026-07-27 14:07:43 +08:00
1 parent 1f01d05603
commit eaf90d8eff
28 files changed
+2615 -133

No files matched your search

@@ -29,6 +29,7 @@ from vllm_omni.diffusion.models.helios.scheduling_helios import HeliosScheduler
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.prompt_update import PromptUpdateMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.platforms import current_omni_platform
@@ -155,7 +156,12 @@ def get_helios_pre_process_func(
class HeliosPipeline(
nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipelineProfilerMixin, SupportsComponentDiscovery
nn.Module,
CFGParallelMixin,
ProgressBarMixin,
DiffusionPipelineProfilerMixin,
PromptUpdateMixin,
SupportsComponentDiscovery,
):
"""Helios text-to-video / image-to-video / video-to-video pipeline for vllm-omni.
@@ -894,9 +900,12 @@ class HeliosPipeline(
output = current_latents if extra["output_type"] == "latent" else current_video
completed_chunk_index = state.chunk_index
prompt_update_metadata = state.extra.pop("prompt_update_chunk_metadata", {})
state.chunk_index += 1
finished = state.request_denoise_completed
if not finished:
# Apply queued/advancing prompt updates before preparing the next chunk.
self._apply_prompt_update_at_chunk_boundary(state)
self._prepare_next_chunk(state)
else:
self._current_timestep = None
@@ -908,6 +917,9 @@ class HeliosPipeline(
stage_durations=self.stage_durations if hasattr(self, "stage_durations") else {},
chunk_index=completed_chunk_index,
total_chunks=state.total_chunks,
started_event_ids=prompt_update_metadata.get("started_event_ids", []),
active_event_ids=prompt_update_metadata.get("active_event_ids", []),
completed_event_ids=prompt_update_metadata.get("completed_event_ids", []),
finished=finished,
)
@@ -1585,12 +1597,13 @@ class HeliosPipeline(
negative_prompt: str | list[str] | None = None,
do_classifier_free_guidance: bool = True,
num_videos_per_prompt: int = 1,
max_sequence_length: int = 226,
max_sequence_length: int | None = None,
device: torch.device | None = None,
dtype: torch.dtype | None = None,
):
device = device or self.device
dtype = dtype or self.text_encoder.dtype
max_sequence_length = max_sequence_length or 226
prompt = [prompt] if isinstance(prompt, str) else prompt
prompt_clean = [self._prompt_clean(p) for p in prompt]
+32 -1
View File
@@ -13,6 +13,8 @@ from typing import (
)
if TYPE_CHECKING:
from collections.abc import Sequence
import torch
from vllm_omni.diffusion.data import DiffusionOutput
@@ -58,7 +60,9 @@ class SupportsStepExecution(Protocol):
"""Prepare request-level inputs and return initialized state."""
...
def denoise_step(self, input_batch: InputBatch, **kwargs: Any) -> torch.Tensor | None:
def denoise_step(
self, input_batch: InputBatch, *, states: Sequence[StepRequestState] | None = None, **kwargs: Any
) -> torch.Tensor | None:
"""Run one denoise forward on the runner-assembled batch."""
...
@@ -100,3 +104,30 @@ def supports_step_execution(pipeline: object) -> bool:
"""Return whether `pipeline` implements :class:`SupportsStepExecution`."""
return isinstance(pipeline, SupportsStepExecution)
@runtime_checkable
class SupportsPromptUpdate(Protocol):
"""Optional protocol for pipelines that support midway prompt updates.
Pipelines typically implement this via
:class:`~vllm_omni.diffusion.prompt_update.PromptUpdateMixin`.
"""
supports_prompt_update: ClassVar[bool] = True
def prepare_prompt_update(
self,
state: StepRequestState,
prompt: str,
event_id: str,
transition_chunks: int | None = None,
) -> None:
"""Encode and queue a prompt update on request-local state."""
...
def supports_prompt_update(pipeline: object) -> bool:
"""Return whether ``pipeline`` implements :class:`SupportsPromptUpdate`."""
return isinstance(pipeline, SupportsPromptUpdate)