[Frontend] Add midway prompt update for streaming video generation (#4652)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
Zeyu Huang | 黃澤宇 authored and GitHub committed 2026-07-27 14:07:43 +08:00
1 parent 1f01d05603
commit eaf90d8eff
28 files changed
+2615 -133

No files matched your search

+3
View File
@@ -1274,6 +1274,9 @@ class DiffusionOutput:
finished: bool = True
chunk_index: int = 0
total_chunks: int = 1
started_event_ids: list[str] = field(default_factory=list)
active_event_ids: list[str] = field(default_factory=list)
completed_event_ids: list[str] = field(default_factory=list)
# logged duration of stages
stage_durations: dict[str, float] = field(default_factory=dict)
@@ -21,7 +21,7 @@ from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.engine.stage_client import StageClientBase
from vllm_omni.engine.stage_init_utils import StageMetadata
from vllm_omni.errors import client_error_metadata
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniInteractionPrompt
from vllm_omni.outputs import OmniRequestOutput
if TYPE_CHECKING:
@@ -179,6 +179,25 @@ class InlineStageDiffusionClient(StageClientBase):
task.cancel()
self._engine.abort(rid)
async def submit_interaction_async(
self,
request_id: str,
interaction: OmniInteractionPrompt,
timeout: float | None = None,
) -> Any:
"""Apply a midway interaction to an active streaming request."""
logger.debug(
"[InlineStageDiffusionClient] stage-%s [rep-%s] interaction: %s",
self.stage_id,
self.replica_id,
request_id,
)
return await self.collective_rpc_async(
"submit_interaction",
timeout=timeout,
args=(request_id, interaction),
)
async def collective_rpc_async(
self,
method: str,
@@ -29,6 +29,7 @@ from vllm_omni.diffusion.models.helios.scheduling_helios import HeliosScheduler
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.prompt_update import PromptUpdateMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.platforms import current_omni_platform
@@ -155,7 +156,12 @@ def get_helios_pre_process_func(
class HeliosPipeline(
nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipelineProfilerMixin, SupportsComponentDiscovery
nn.Module,
CFGParallelMixin,
ProgressBarMixin,
DiffusionPipelineProfilerMixin,
PromptUpdateMixin,
SupportsComponentDiscovery,
):
"""Helios text-to-video / image-to-video / video-to-video pipeline for vllm-omni.
@@ -894,9 +900,12 @@ class HeliosPipeline(
output = current_latents if extra["output_type"] == "latent" else current_video
completed_chunk_index = state.chunk_index
prompt_update_metadata = state.extra.pop("prompt_update_chunk_metadata", {})
state.chunk_index += 1
finished = state.request_denoise_completed
if not finished:
# Apply queued/advancing prompt updates before preparing the next chunk.
self._apply_prompt_update_at_chunk_boundary(state)
self._prepare_next_chunk(state)
else:
self._current_timestep = None
@@ -908,6 +917,9 @@ class HeliosPipeline(
stage_durations=self.stage_durations if hasattr(self, "stage_durations") else {},
chunk_index=completed_chunk_index,
total_chunks=state.total_chunks,
started_event_ids=prompt_update_metadata.get("started_event_ids", []),
active_event_ids=prompt_update_metadata.get("active_event_ids", []),
completed_event_ids=prompt_update_metadata.get("completed_event_ids", []),
finished=finished,
)
@@ -1585,12 +1597,13 @@ class HeliosPipeline(
negative_prompt: str | list[str] | None = None,
do_classifier_free_guidance: bool = True,
num_videos_per_prompt: int = 1,
max_sequence_length: int = 226,
max_sequence_length: int | None = None,
device: torch.device | None = None,
dtype: torch.dtype | None = None,
):
device = device or self.device
dtype = dtype or self.text_encoder.dtype
max_sequence_length = max_sequence_length or 226
prompt = [prompt] if isinstance(prompt, str) else prompt
prompt_clean = [self._prompt_clean(p) for p in prompt]
+32 -1
View File
@@ -13,6 +13,8 @@ from typing import (
)
if TYPE_CHECKING:
from collections.abc import Sequence
import torch
from vllm_omni.diffusion.data import DiffusionOutput
@@ -58,7 +60,9 @@ class SupportsStepExecution(Protocol):
"""Prepare request-level inputs and return initialized state."""
...
def denoise_step(self, input_batch: InputBatch, **kwargs: Any) -> torch.Tensor | None:
def denoise_step(
self, input_batch: InputBatch, *, states: Sequence[StepRequestState] | None = None, **kwargs: Any
) -> torch.Tensor | None:
"""Run one denoise forward on the runner-assembled batch."""
...
@@ -100,3 +104,30 @@ def supports_step_execution(pipeline: object) -> bool:
"""Return whether `pipeline` implements :class:`SupportsStepExecution`."""
return isinstance(pipeline, SupportsStepExecution)
@runtime_checkable
class SupportsPromptUpdate(Protocol):
"""Optional protocol for pipelines that support midway prompt updates.
Pipelines typically implement this via
:class:`~vllm_omni.diffusion.prompt_update.PromptUpdateMixin`.
"""
supports_prompt_update: ClassVar[bool] = True
def prepare_prompt_update(
self,
state: StepRequestState,
prompt: str,
event_id: str,
transition_chunks: int | None = None,
) -> None:
"""Encode and queue a prompt update on request-local state."""
...
def supports_prompt_update(pipeline: object) -> bool:
"""Return whether ``pipeline`` implements :class:`SupportsPromptUpdate`."""
return isinstance(pipeline, SupportsPromptUpdate)
+29 -1
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import TypeGuard
from typing import TypeGuard, cast
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.io_support import supports_audio_output
@@ -150,6 +150,13 @@ def format_diffusion_outputs(
model_cls = DiffusionModelRegistry._try_load_model_cls(od_config.model_class_name)
audio_sample_rate = getattr(model_cls, "audio_sample_rate", None)
stream_metadata = _stream_metadata_from_diffusion_output(diffusion_output)
if stream_metadata:
existing_stream = postprocess_output.metadata.get("stream")
merged_stream = dict(existing_stream) if isinstance(existing_stream, Mapping) else {}
merged_stream.update(stream_metadata)
postprocess_output.metadata["stream"] = merged_stream
return _format_single_prompt_output(
request=request,
prompt=request.prompt,
@@ -164,6 +171,27 @@ def format_diffusion_outputs(
)
def _stream_metadata_from_diffusion_output(diffusion_output: DiffusionOutput) -> DiffusionMetadata | None:
has_stream_data = (
diffusion_output.total_chunks > 1
or not diffusion_output.finished
or bool(diffusion_output.started_event_ids)
or bool(diffusion_output.active_event_ids)
or bool(diffusion_output.completed_event_ids)
)
if not has_stream_data:
return None
return cast(
DiffusionMetadata,
{
"generation_chunk_index": diffusion_output.chunk_index,
"started_event_ids": diffusion_output.started_event_ids,
"active_event_ids": diffusion_output.active_event_ids,
"completed_event_ids": diffusion_output.completed_event_ids,
},
)
def _ensure_list(outputs: DiffusionPayloadValue) -> list[DiffusionPayloadValue]:
if isinstance(outputs, list):
return outputs
+245
View File
@@ -0,0 +1,245 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Midway prompt updates for chunked diffusion step execution.
``PromptUpdateMixin`` exposes
- a pipeline interface for the external to queue a prompt update,
- a pipeline internal method to apply prompt transitions at chunk boundaries.
``prompt_update_versions``: a helper for ``InputBatch`` cache invalidation.
"""
from __future__ import annotations
import logging
from collections.abc import Sequence
from dataclasses import dataclass
from threading import Lock
from typing import ClassVar, TypedDict, cast
import torch
from vllm_omni.diffusion.worker.utils import StepRequestState
DEFAULT_TRANSITION_CHUNKS = 3
logger = logging.getLogger(__name__)
class PromptUpdateExtra(TypedDict, total=False):
"""A "protocol" for ``StepRequestState.extra`` keys used by the prompt-update feature."""
pending_prompt_update: _PendingPromptUpdate
prompt_update_state: _PromptUpdateState
prompt_update_chunk_metadata: _PromptUpdateChunkMetadata
prompt_update_version: int # Bumped when prompt_embeds change; invalidates InputBatch cache.
# The lock for storing/updating pending prompt updates.
# Saved in state.extra to be naturally cleaned up with the request state.
# As PromptUpdateMixin (the pipeline) currently lacks an obvious request-finish hook
prompt_update_lock: Lock
def prompt_update_versions(states: Sequence[StepRequestState]) -> tuple[int, ...]:
"""Return per-request prompt-update versions for batch cache comparison.
To be used by ``InputBatch`` or other external to determine if the cached batch is still valid.
"""
return tuple(int(cast(PromptUpdateExtra, state.extra).get("prompt_update_version", 0)) for state in states)
class PromptUpdateMixin:
"""Mixin for chunked streaming pipelines that support midway prompt updates.
Implements the behavior expected by :class:`~vllm_omni.diffusion.models.interface.SupportsPromptUpdate`.
All per-request transition state lives on ``StepRequestState.extra``; this mixin itself is stateless.
"""
supports_prompt_update: ClassVar[bool] = True
@staticmethod
def _prompt_update_lock(extra: PromptUpdateExtra) -> Lock:
return extra.setdefault("prompt_update_lock", Lock())
def prepare_prompt_update(
self,
state: StepRequestState,
prompt: str,
event_id: str,
transition_chunks: int | None = None,
) -> None:
"""Encode and queue a prompt update to apply at the next chunk boundary.
It does not actually apply the prompt transition.
This is an extended interface for the pipeline, to be called by the runner.
"""
if not prompt:
raise ValueError("prompt must be non-empty")
if not event_id:
raise ValueError("event_id must be non-empty")
if state.prompt_embeds is None:
raise ValueError(
f"prompt_update is not allowed before initial generation has started (request_id={state.request_id!r})"
)
duration = DEFAULT_TRANSITION_CHUNKS if transition_chunks is None else transition_chunks
if duration < 0:
raise ValueError("transition_chunks must be >= 0")
target_prompt_embeds, _ = self.encode_prompt(
prompt=prompt,
negative_prompt=None,
do_classifier_free_guidance=False,
num_videos_per_prompt=state.sampling.num_outputs_per_prompt,
max_sequence_length=state.sampling.max_sequence_length,
device=self.device,
dtype=self.transformer.dtype,
)
extra = cast(PromptUpdateExtra, state.extra)
with self._prompt_update_lock(extra):
extra["pending_prompt_update"] = {
"event_id": event_id,
"prompt": prompt,
"target_prompt_embeds": target_prompt_embeds,
"transition_chunks": duration,
}
def _apply_prompt_update_at_chunk_boundary(self, state: StepRequestState) -> None:
"""Advance or start prompt interpolation before the next chunk."""
extra = cast(PromptUpdateExtra, state.extra)
update_state: _PromptUpdateState | None = extra.get("prompt_update_state")
with self._prompt_update_lock(extra):
pending = extra.pop("pending_prompt_update", None)
embeds_changed = False
next_chunk_index = state.chunk_index
started_event_ids: list[str] = []
active_event_ids: list[str] = []
completed_event_ids: list[str] = []
# If current transition is not complete, advance it.
# After completion, leave prompt_update_state in place (so a later pending
# update can abort/overwrite), but do not keep bumping the version.
if update_state is not None:
in_transition = (
update_state.transition_chunks > 0
and update_state.elapsed_transition_chunks < update_state.transition_chunks
)
if in_transition:
update_state.advance_transition()
state.prompt_embeds = update_state.blended_prompt_embeds()
embeds_changed = True
active_event_ids.append(update_state.event_id)
if update_state.elapsed_transition_chunks >= update_state.transition_chunks:
state.prompt_embeds = update_state.target_prompt_embeds
update_state.source_prompt_embeds = update_state.target_prompt_embeds
completed_event_ids.append(update_state.event_id)
if update_state.target_prompt is not None:
logger.debug(
"prompt_update transition complete request_id=%s next_chunk_index=%d prompt=%.20s...",
state.request_id,
next_chunk_index,
update_state.target_prompt,
)
# If a new prompt update is pending, start a new transition.
if pending is not None:
if state.prompt_embeds is None:
raise RuntimeError(
"internal error: trying to apply a pending prompt update but "
f"current prompt_embeds is None (request_id={state.request_id!r})"
)
source = state.prompt_embeds.detach().clone()
target = pending["target_prompt_embeds"]
duration = int(pending["transition_chunks"])
prompt = str(pending.get("prompt", ""))
event_id = pending.get("event_id")
update_state = _PromptUpdateState(
source_prompt_embeds=source,
target_prompt_embeds=target,
transition_chunks=duration,
event_id=event_id,
target_prompt=prompt,
)
extra["prompt_update_state"] = update_state
started_event_ids.append(event_id)
active_event_ids.append(event_id)
if duration <= 0:
# A hard/immediate transition
state.prompt_embeds = target
completed_event_ids.append(event_id)
logger.debug(
"prompt_update sharp transition request_id=%s next_chunk_index=%d prompt=%.20s...",
state.request_id,
next_chunk_index,
prompt,
)
else:
# A transition that really takes some time
update_state.advance_transition()
state.prompt_embeds = update_state.blended_prompt_embeds()
if update_state.elapsed_transition_chunks >= update_state.transition_chunks:
state.prompt_embeds = update_state.target_prompt_embeds
update_state.source_prompt_embeds = update_state.target_prompt_embeds
completed_event_ids.append(event_id)
logger.debug(
"prompt_update transition start request_id=%s next_chunk_index=%d prompt=%.20s...",
state.request_id,
next_chunk_index,
prompt,
)
embeds_changed = True
# Indicate that prompt embeddings have changed---clear input batch cache.
if embeds_changed:
new_version = int(extra.get("prompt_update_version", 0)) + 1
extra["prompt_update_version"] = new_version
extra["prompt_update_chunk_metadata"] = {
"started_event_ids": started_event_ids,
"active_event_ids": active_event_ids,
"completed_event_ids": completed_event_ids,
}
@dataclass
class _PromptUpdateState:
"""Per-request prompt interpolation stored in ``state.extra["prompt_update_state"]``."""
source_prompt_embeds: torch.Tensor
target_prompt_embeds: torch.Tensor
transition_chunks: int
event_id: str
elapsed_transition_chunks: int = 0
target_prompt: str | None = None
def blended_prompt_embeds(self) -> torch.Tensor:
alpha = self._current_alpha()
if alpha >= 1.0:
return self.target_prompt_embeds
return (1.0 - alpha) * self.source_prompt_embeds + alpha * self.target_prompt_embeds
def advance_transition(self) -> None:
if self.transition_chunks <= 0:
self.source_prompt_embeds = self.target_prompt_embeds
return
self.elapsed_transition_chunks += 1
if self.elapsed_transition_chunks >= self.transition_chunks:
self.source_prompt_embeds = self.target_prompt_embeds
self.elapsed_transition_chunks = self.transition_chunks
def _current_alpha(self) -> float:
if self.transition_chunks <= 0:
return 1.0
return min(1.0, self.elapsed_transition_chunks / self.transition_chunks)
class _PendingPromptUpdate(TypedDict):
event_id: str
prompt: str
target_prompt_embeds: torch.Tensor
transition_chunks: int
class _PromptUpdateChunkMetadata(TypedDict):
started_event_ids: list[str]
active_event_ids: list[str]
completed_event_ids: list[str]
+14 -1
View File
@@ -34,7 +34,7 @@ from vllm_omni.outputs import OmniRequestOutput
if TYPE_CHECKING:
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniInteractionPrompt, OmniPromptType
logger = init_logger(__name__)
_MISSING_RPC_RESULT = object()
@@ -395,6 +395,19 @@ class StageDiffusionClient(StageClientBase):
)
)
async def submit_interaction_async(
self,
request_id: str,
interaction: OmniInteractionPrompt,
timeout: float | None = None,
) -> Any:
"""Apply a midway interaction to an active streaming request."""
return await self.collective_rpc_async(
"submit_interaction",
timeout=timeout,
args=(request_id, interaction),
)
async def collective_rpc_async(
self,
method: str,
@@ -14,7 +14,7 @@ import copy
import time
from collections.abc import Callable
from contextlib import AbstractContextManager, nullcontext
from typing import Any
from typing import TYPE_CHECKING, Any, cast
import torch
from torch.profiler import record_function
@@ -32,7 +32,7 @@ from vllm_omni.diffusion.compile import regionally_compile
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.forward_context import set_forward_context
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import supports_step_execution
from vllm_omni.diffusion.models.interface import SupportsPromptUpdate, supports_prompt_update, supports_step_execution
from vllm_omni.diffusion.offloader import get_offload_backend
from vllm_omni.diffusion.registry import _NO_CACHE_ACCELERATION
from vllm_omni.diffusion.request import OmniDiffusionRequest
@@ -53,6 +53,9 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
from vllm_omni.worker.omni_connector_model_runner_mixin import OmniConnectorModelRunnerMixin
if TYPE_CHECKING:
from vllm_omni.inputs.data import OmniInteractionPrompt
logger = init_logger(__name__)
@@ -803,3 +806,48 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
self._update_states_after(states, input_batch, pipeline_interrupted)
return BatchRunnerOutput.from_list(runner_output_list)
def submit_interaction(
self,
request_id: str,
interaction: OmniInteractionPrompt,
) -> None:
"""Route a midway interaction to the matching active stepwise request feature."""
assert self.pipeline is not None, "Model not loaded. Call load_model() first."
if not self.od_config.streaming_output:
raise ValueError("submit_interaction requires streaming_output=True")
if not self._supports_step_mode():
raise ValueError("submit_interaction requires step execution support")
event = interaction.get("event")
if isinstance(event, dict) and "prompt" in event and "multi_modal_data" not in event:
# Is a prompt update interaction.
self._submit_prompt_update_interaction(request_id, interaction)
return
raise NotImplementedError(
"Only text-only prompt update interactions with 'event.prompt' and optional "
"'transition_chunks' are supported in this release"
)
def _submit_prompt_update_interaction(
self,
request_id: str,
interaction: OmniInteractionPrompt,
) -> None:
"""Queue a prompt-update interaction for an active stepwise request."""
if not supports_prompt_update(self.pipeline):
raise ValueError(f"prompt_update is not supported by pipeline {self.od_config.model_class_name!r}")
state = self.state_cache.get(request_id)
if state is None:
raise ValueError(f"No active request state for prompt_update: {request_id!r}")
event = cast(dict[str, Any], interaction.get("event"))
prompt = event["prompt"]
transition_chunks = interaction.get("transition_chunks")
event_id = interaction.get("event_id")
if not isinstance(event_id, str) or not event_id:
raise ValueError("event_id must be non-empty")
pipeline = cast(SupportsPromptUpdate, self.pipeline)
pipeline.prepare_prompt_update(state, prompt, event_id, transition_chunks)
@@ -50,6 +50,7 @@ from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput, KVPref
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
from vllm_omni.diffusion.worker.utils import BaseRunnerOutput, BatchRunnerOutput
from vllm_omni.engine.stage_init_utils import set_death_signal
from vllm_omni.inputs.data import OmniInteractionPrompt
from vllm_omni.lora.request import LoRARequest
from vllm_omni.platforms import current_omni_platform
from vllm_omni.profiler import OmniTorchProfilerWrapper, create_omni_profiler
@@ -511,6 +512,15 @@ class DiffusionWorker:
# for the frontend server yet.
return self.lora_manager.add_adapter(lora_request)
def submit_interaction(
self,
request_id: str,
interaction: OmniInteractionPrompt,
) -> None:
"""Apply a midway interaction to an active stepwise request."""
assert self.model_runner is not None, "Model runner not initialized"
self.model_runner.submit_interaction(request_id, interaction)
def list_loras(self) -> list[int]:
return self.lora_manager.list_adapters()
+14 -2
View File
@@ -17,6 +17,7 @@ from dataclasses import dataclass, field
import numpy as np
import torch
from vllm_omni.diffusion.prompt_update import prompt_update_versions
from vllm_omni.diffusion.worker.utils import StepRequestState
@@ -535,12 +536,17 @@ def _same_composition(
cached_batch: InputBatch | None,
request_ids: list[str],
idx_mapping_np: np.ndarray,
states: Sequence[StepRequestState],
) -> bool:
if cached_batch is None:
return False
if cached_batch.request_ids != request_ids:
return False
return np.array_equal(cached_batch.idx_mapping_np, idx_mapping_np)
if not np.array_equal(cached_batch.idx_mapping_np, idx_mapping_np):
return False
# Midway prompt updates (typically for video generation) can change prompt_embeds without changing ids/mapping.
# In this case, each request state manages the embedding's "version". Use it to determine if cache is still valid.
return cached_batch._prompt_update_versions == prompt_update_versions(states)
def _scatter_batch_tensor_by_mapping(
@@ -615,6 +621,10 @@ class InputBatch:
negative_txt_seq_lens: list[int] | None = None
states: Sequence[StepRequestState] = field(default_factory=tuple)
# For midway prompt updates (typically for video generation) that changes embeddings without changing ids,
# Keep a snapshot of the current per-request versions of prompt embeddings for later runtime comparison
_prompt_update_versions: tuple[int, ...] = ()
def __post_init__(self) -> None:
if len(self.request_ids) != int(self.idx_mapping.numel()):
raise ValueError("`request_ids` and `idx_mapping` must have the same length.")
@@ -653,6 +663,7 @@ class InputBatch:
self.img_shapes = _prepare_img_shapes(states)
self.txt_seq_lens = _prepare_seq_lens(states, "txt_seq_lens")
self.negative_txt_seq_lens = _prepare_seq_lens(states, "negative_txt_seq_lens")
self._prompt_update_versions = prompt_update_versions(states)
def _repack_dynamic_fields(
self,
@@ -693,7 +704,7 @@ class InputBatch:
selected_states, idx_mapping, idx_mapping_np = _select_states(states, idx_mapping)
request_ids = _prepare_request_ids(selected_states)
if _same_composition(cached_batch, request_ids, idx_mapping_np):
if _same_composition(cached_batch, request_ids, idx_mapping_np, selected_states):
assert cached_batch is not None
cached_batch._repack_dynamic_fields(selected_states)
return cached_batch
@@ -733,6 +744,7 @@ class InputBatch:
"negative_txt_seq_lens",
),
states=tuple(selected_states),
_prompt_update_versions=prompt_update_versions(selected_states),
)
@@ -48,6 +48,9 @@ def split_diffusion_output_by_request(
peak_memory_mb=result.peak_memory_mb,
chunk_index=result.chunk_index,
total_chunks=result.total_chunks,
started_event_ids=result.started_event_ids,
active_event_ids=result.active_event_ids,
completed_event_ids=result.completed_event_ids,
)
for idx in range(req.num_reqs)
]