[Frontend] Add midway prompt update for streaming video generation (#4652)
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
1 parent
1f01d05603
commit
eaf90d8eff
28 files changed
+2615
-133
No files matched your search
@@ -1274,6 +1274,9 @@ class DiffusionOutput:
|
||||
finished: bool = True
|
||||
chunk_index: int = 0
|
||||
total_chunks: int = 1
|
||||
started_event_ids: list[str] = field(default_factory=list)
|
||||
active_event_ids: list[str] = field(default_factory=list)
|
||||
completed_event_ids: list[str] = field(default_factory=list)
|
||||
|
||||
# logged duration of stages
|
||||
stage_durations: dict[str, float] = field(default_factory=dict)
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.engine.stage_client import StageClientBase
|
||||
from vllm_omni.engine.stage_init_utils import StageMetadata
|
||||
from vllm_omni.errors import client_error_metadata
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniInteractionPrompt
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -179,6 +179,25 @@ class InlineStageDiffusionClient(StageClientBase):
|
||||
task.cancel()
|
||||
self._engine.abort(rid)
|
||||
|
||||
async def submit_interaction_async(
|
||||
self,
|
||||
request_id: str,
|
||||
interaction: OmniInteractionPrompt,
|
||||
timeout: float | None = None,
|
||||
) -> Any:
|
||||
"""Apply a midway interaction to an active streaming request."""
|
||||
logger.debug(
|
||||
"[InlineStageDiffusionClient] stage-%s [rep-%s] interaction: %s",
|
||||
self.stage_id,
|
||||
self.replica_id,
|
||||
request_id,
|
||||
)
|
||||
return await self.collective_rpc_async(
|
||||
"submit_interaction",
|
||||
timeout=timeout,
|
||||
args=(request_id, interaction),
|
||||
)
|
||||
|
||||
async def collective_rpc_async(
|
||||
self,
|
||||
method: str,
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm_omni.diffusion.models.helios.scheduling_helios import HeliosScheduler
|
||||
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
|
||||
from vllm_omni.diffusion.prompt_update import PromptUpdateMixin
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
@@ -155,7 +156,12 @@ def get_helios_pre_process_func(
|
||||
|
||||
|
||||
class HeliosPipeline(
|
||||
nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipelineProfilerMixin, SupportsComponentDiscovery
|
||||
nn.Module,
|
||||
CFGParallelMixin,
|
||||
ProgressBarMixin,
|
||||
DiffusionPipelineProfilerMixin,
|
||||
PromptUpdateMixin,
|
||||
SupportsComponentDiscovery,
|
||||
):
|
||||
"""Helios text-to-video / image-to-video / video-to-video pipeline for vllm-omni.
|
||||
|
||||
@@ -894,9 +900,12 @@ class HeliosPipeline(
|
||||
|
||||
output = current_latents if extra["output_type"] == "latent" else current_video
|
||||
completed_chunk_index = state.chunk_index
|
||||
prompt_update_metadata = state.extra.pop("prompt_update_chunk_metadata", {})
|
||||
state.chunk_index += 1
|
||||
finished = state.request_denoise_completed
|
||||
if not finished:
|
||||
# Apply queued/advancing prompt updates before preparing the next chunk.
|
||||
self._apply_prompt_update_at_chunk_boundary(state)
|
||||
self._prepare_next_chunk(state)
|
||||
else:
|
||||
self._current_timestep = None
|
||||
@@ -908,6 +917,9 @@ class HeliosPipeline(
|
||||
stage_durations=self.stage_durations if hasattr(self, "stage_durations") else {},
|
||||
chunk_index=completed_chunk_index,
|
||||
total_chunks=state.total_chunks,
|
||||
started_event_ids=prompt_update_metadata.get("started_event_ids", []),
|
||||
active_event_ids=prompt_update_metadata.get("active_event_ids", []),
|
||||
completed_event_ids=prompt_update_metadata.get("completed_event_ids", []),
|
||||
finished=finished,
|
||||
)
|
||||
|
||||
@@ -1585,12 +1597,13 @@ class HeliosPipeline(
|
||||
negative_prompt: str | list[str] | None = None,
|
||||
do_classifier_free_guidance: bool = True,
|
||||
num_videos_per_prompt: int = 1,
|
||||
max_sequence_length: int = 226,
|
||||
max_sequence_length: int | None = None,
|
||||
device: torch.device | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
):
|
||||
device = device or self.device
|
||||
dtype = dtype or self.text_encoder.dtype
|
||||
max_sequence_length = max_sequence_length or 226
|
||||
|
||||
prompt = [prompt] if isinstance(prompt, str) else prompt
|
||||
prompt_clean = [self._prompt_clean(p) for p in prompt]
|
||||
|
||||
@@ -13,6 +13,8 @@ from typing import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput
|
||||
@@ -58,7 +60,9 @@ class SupportsStepExecution(Protocol):
|
||||
"""Prepare request-level inputs and return initialized state."""
|
||||
...
|
||||
|
||||
def denoise_step(self, input_batch: InputBatch, **kwargs: Any) -> torch.Tensor | None:
|
||||
def denoise_step(
|
||||
self, input_batch: InputBatch, *, states: Sequence[StepRequestState] | None = None, **kwargs: Any
|
||||
) -> torch.Tensor | None:
|
||||
"""Run one denoise forward on the runner-assembled batch."""
|
||||
...
|
||||
|
||||
@@ -100,3 +104,30 @@ def supports_step_execution(pipeline: object) -> bool:
|
||||
"""Return whether `pipeline` implements :class:`SupportsStepExecution`."""
|
||||
|
||||
return isinstance(pipeline, SupportsStepExecution)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsPromptUpdate(Protocol):
|
||||
"""Optional protocol for pipelines that support midway prompt updates.
|
||||
|
||||
Pipelines typically implement this via
|
||||
:class:`~vllm_omni.diffusion.prompt_update.PromptUpdateMixin`.
|
||||
"""
|
||||
|
||||
supports_prompt_update: ClassVar[bool] = True
|
||||
|
||||
def prepare_prompt_update(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
prompt: str,
|
||||
event_id: str,
|
||||
transition_chunks: int | None = None,
|
||||
) -> None:
|
||||
"""Encode and queue a prompt update on request-local state."""
|
||||
...
|
||||
|
||||
|
||||
def supports_prompt_update(pipeline: object) -> bool:
|
||||
"""Return whether ``pipeline`` implements :class:`SupportsPromptUpdate`."""
|
||||
|
||||
return isinstance(pipeline, SupportsPromptUpdate)
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TypeGuard
|
||||
from typing import TypeGuard, cast
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.io_support import supports_audio_output
|
||||
@@ -150,6 +150,13 @@ def format_diffusion_outputs(
|
||||
model_cls = DiffusionModelRegistry._try_load_model_cls(od_config.model_class_name)
|
||||
audio_sample_rate = getattr(model_cls, "audio_sample_rate", None)
|
||||
|
||||
stream_metadata = _stream_metadata_from_diffusion_output(diffusion_output)
|
||||
if stream_metadata:
|
||||
existing_stream = postprocess_output.metadata.get("stream")
|
||||
merged_stream = dict(existing_stream) if isinstance(existing_stream, Mapping) else {}
|
||||
merged_stream.update(stream_metadata)
|
||||
postprocess_output.metadata["stream"] = merged_stream
|
||||
|
||||
return _format_single_prompt_output(
|
||||
request=request,
|
||||
prompt=request.prompt,
|
||||
@@ -164,6 +171,27 @@ def format_diffusion_outputs(
|
||||
)
|
||||
|
||||
|
||||
def _stream_metadata_from_diffusion_output(diffusion_output: DiffusionOutput) -> DiffusionMetadata | None:
|
||||
has_stream_data = (
|
||||
diffusion_output.total_chunks > 1
|
||||
or not diffusion_output.finished
|
||||
or bool(diffusion_output.started_event_ids)
|
||||
or bool(diffusion_output.active_event_ids)
|
||||
or bool(diffusion_output.completed_event_ids)
|
||||
)
|
||||
if not has_stream_data:
|
||||
return None
|
||||
return cast(
|
||||
DiffusionMetadata,
|
||||
{
|
||||
"generation_chunk_index": diffusion_output.chunk_index,
|
||||
"started_event_ids": diffusion_output.started_event_ids,
|
||||
"active_event_ids": diffusion_output.active_event_ids,
|
||||
"completed_event_ids": diffusion_output.completed_event_ids,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _ensure_list(outputs: DiffusionPayloadValue) -> list[DiffusionPayloadValue]:
|
||||
if isinstance(outputs, list):
|
||||
return outputs
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Midway prompt updates for chunked diffusion step execution.
|
||||
|
||||
``PromptUpdateMixin`` exposes
|
||||
- a pipeline interface for the external to queue a prompt update,
|
||||
- a pipeline internal method to apply prompt transitions at chunk boundaries.
|
||||
|
||||
``prompt_update_versions``: a helper for ``InputBatch`` cache invalidation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from threading import Lock
|
||||
from typing import ClassVar, TypedDict, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
DEFAULT_TRANSITION_CHUNKS = 3
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PromptUpdateExtra(TypedDict, total=False):
|
||||
"""A "protocol" for ``StepRequestState.extra`` keys used by the prompt-update feature."""
|
||||
|
||||
pending_prompt_update: _PendingPromptUpdate
|
||||
prompt_update_state: _PromptUpdateState
|
||||
prompt_update_chunk_metadata: _PromptUpdateChunkMetadata
|
||||
prompt_update_version: int # Bumped when prompt_embeds change; invalidates InputBatch cache.
|
||||
|
||||
# The lock for storing/updating pending prompt updates.
|
||||
# Saved in state.extra to be naturally cleaned up with the request state.
|
||||
# As PromptUpdateMixin (the pipeline) currently lacks an obvious request-finish hook
|
||||
prompt_update_lock: Lock
|
||||
|
||||
|
||||
def prompt_update_versions(states: Sequence[StepRequestState]) -> tuple[int, ...]:
|
||||
"""Return per-request prompt-update versions for batch cache comparison.
|
||||
|
||||
To be used by ``InputBatch`` or other external to determine if the cached batch is still valid.
|
||||
"""
|
||||
return tuple(int(cast(PromptUpdateExtra, state.extra).get("prompt_update_version", 0)) for state in states)
|
||||
|
||||
|
||||
class PromptUpdateMixin:
|
||||
"""Mixin for chunked streaming pipelines that support midway prompt updates.
|
||||
|
||||
Implements the behavior expected by :class:`~vllm_omni.diffusion.models.interface.SupportsPromptUpdate`.
|
||||
All per-request transition state lives on ``StepRequestState.extra``; this mixin itself is stateless.
|
||||
"""
|
||||
|
||||
supports_prompt_update: ClassVar[bool] = True
|
||||
|
||||
@staticmethod
|
||||
def _prompt_update_lock(extra: PromptUpdateExtra) -> Lock:
|
||||
return extra.setdefault("prompt_update_lock", Lock())
|
||||
|
||||
def prepare_prompt_update(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
prompt: str,
|
||||
event_id: str,
|
||||
transition_chunks: int | None = None,
|
||||
) -> None:
|
||||
"""Encode and queue a prompt update to apply at the next chunk boundary.
|
||||
It does not actually apply the prompt transition.
|
||||
|
||||
This is an extended interface for the pipeline, to be called by the runner.
|
||||
"""
|
||||
if not prompt:
|
||||
raise ValueError("prompt must be non-empty")
|
||||
if not event_id:
|
||||
raise ValueError("event_id must be non-empty")
|
||||
if state.prompt_embeds is None:
|
||||
raise ValueError(
|
||||
f"prompt_update is not allowed before initial generation has started (request_id={state.request_id!r})"
|
||||
)
|
||||
duration = DEFAULT_TRANSITION_CHUNKS if transition_chunks is None else transition_chunks
|
||||
if duration < 0:
|
||||
raise ValueError("transition_chunks must be >= 0")
|
||||
|
||||
target_prompt_embeds, _ = self.encode_prompt(
|
||||
prompt=prompt,
|
||||
negative_prompt=None,
|
||||
do_classifier_free_guidance=False,
|
||||
num_videos_per_prompt=state.sampling.num_outputs_per_prompt,
|
||||
max_sequence_length=state.sampling.max_sequence_length,
|
||||
device=self.device,
|
||||
dtype=self.transformer.dtype,
|
||||
)
|
||||
extra = cast(PromptUpdateExtra, state.extra)
|
||||
with self._prompt_update_lock(extra):
|
||||
extra["pending_prompt_update"] = {
|
||||
"event_id": event_id,
|
||||
"prompt": prompt,
|
||||
"target_prompt_embeds": target_prompt_embeds,
|
||||
"transition_chunks": duration,
|
||||
}
|
||||
|
||||
def _apply_prompt_update_at_chunk_boundary(self, state: StepRequestState) -> None:
|
||||
"""Advance or start prompt interpolation before the next chunk."""
|
||||
extra = cast(PromptUpdateExtra, state.extra)
|
||||
update_state: _PromptUpdateState | None = extra.get("prompt_update_state")
|
||||
with self._prompt_update_lock(extra):
|
||||
pending = extra.pop("pending_prompt_update", None)
|
||||
embeds_changed = False
|
||||
next_chunk_index = state.chunk_index
|
||||
started_event_ids: list[str] = []
|
||||
active_event_ids: list[str] = []
|
||||
completed_event_ids: list[str] = []
|
||||
|
||||
# If current transition is not complete, advance it.
|
||||
# After completion, leave prompt_update_state in place (so a later pending
|
||||
# update can abort/overwrite), but do not keep bumping the version.
|
||||
if update_state is not None:
|
||||
in_transition = (
|
||||
update_state.transition_chunks > 0
|
||||
and update_state.elapsed_transition_chunks < update_state.transition_chunks
|
||||
)
|
||||
if in_transition:
|
||||
update_state.advance_transition()
|
||||
state.prompt_embeds = update_state.blended_prompt_embeds()
|
||||
embeds_changed = True
|
||||
active_event_ids.append(update_state.event_id)
|
||||
if update_state.elapsed_transition_chunks >= update_state.transition_chunks:
|
||||
state.prompt_embeds = update_state.target_prompt_embeds
|
||||
update_state.source_prompt_embeds = update_state.target_prompt_embeds
|
||||
completed_event_ids.append(update_state.event_id)
|
||||
if update_state.target_prompt is not None:
|
||||
logger.debug(
|
||||
"prompt_update transition complete request_id=%s next_chunk_index=%d prompt=%.20s...",
|
||||
state.request_id,
|
||||
next_chunk_index,
|
||||
update_state.target_prompt,
|
||||
)
|
||||
|
||||
# If a new prompt update is pending, start a new transition.
|
||||
if pending is not None:
|
||||
if state.prompt_embeds is None:
|
||||
raise RuntimeError(
|
||||
"internal error: trying to apply a pending prompt update but "
|
||||
f"current prompt_embeds is None (request_id={state.request_id!r})"
|
||||
)
|
||||
source = state.prompt_embeds.detach().clone()
|
||||
target = pending["target_prompt_embeds"]
|
||||
duration = int(pending["transition_chunks"])
|
||||
prompt = str(pending.get("prompt", ""))
|
||||
event_id = pending.get("event_id")
|
||||
update_state = _PromptUpdateState(
|
||||
source_prompt_embeds=source,
|
||||
target_prompt_embeds=target,
|
||||
transition_chunks=duration,
|
||||
event_id=event_id,
|
||||
target_prompt=prompt,
|
||||
)
|
||||
extra["prompt_update_state"] = update_state
|
||||
started_event_ids.append(event_id)
|
||||
active_event_ids.append(event_id)
|
||||
if duration <= 0:
|
||||
# A hard/immediate transition
|
||||
state.prompt_embeds = target
|
||||
completed_event_ids.append(event_id)
|
||||
logger.debug(
|
||||
"prompt_update sharp transition request_id=%s next_chunk_index=%d prompt=%.20s...",
|
||||
state.request_id,
|
||||
next_chunk_index,
|
||||
prompt,
|
||||
)
|
||||
else:
|
||||
# A transition that really takes some time
|
||||
update_state.advance_transition()
|
||||
state.prompt_embeds = update_state.blended_prompt_embeds()
|
||||
if update_state.elapsed_transition_chunks >= update_state.transition_chunks:
|
||||
state.prompt_embeds = update_state.target_prompt_embeds
|
||||
update_state.source_prompt_embeds = update_state.target_prompt_embeds
|
||||
completed_event_ids.append(event_id)
|
||||
logger.debug(
|
||||
"prompt_update transition start request_id=%s next_chunk_index=%d prompt=%.20s...",
|
||||
state.request_id,
|
||||
next_chunk_index,
|
||||
prompt,
|
||||
)
|
||||
embeds_changed = True
|
||||
|
||||
# Indicate that prompt embeddings have changed---clear input batch cache.
|
||||
if embeds_changed:
|
||||
new_version = int(extra.get("prompt_update_version", 0)) + 1
|
||||
extra["prompt_update_version"] = new_version
|
||||
|
||||
extra["prompt_update_chunk_metadata"] = {
|
||||
"started_event_ids": started_event_ids,
|
||||
"active_event_ids": active_event_ids,
|
||||
"completed_event_ids": completed_event_ids,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class _PromptUpdateState:
|
||||
"""Per-request prompt interpolation stored in ``state.extra["prompt_update_state"]``."""
|
||||
|
||||
source_prompt_embeds: torch.Tensor
|
||||
target_prompt_embeds: torch.Tensor
|
||||
transition_chunks: int
|
||||
event_id: str
|
||||
elapsed_transition_chunks: int = 0
|
||||
target_prompt: str | None = None
|
||||
|
||||
def blended_prompt_embeds(self) -> torch.Tensor:
|
||||
alpha = self._current_alpha()
|
||||
if alpha >= 1.0:
|
||||
return self.target_prompt_embeds
|
||||
return (1.0 - alpha) * self.source_prompt_embeds + alpha * self.target_prompt_embeds
|
||||
|
||||
def advance_transition(self) -> None:
|
||||
if self.transition_chunks <= 0:
|
||||
self.source_prompt_embeds = self.target_prompt_embeds
|
||||
return
|
||||
self.elapsed_transition_chunks += 1
|
||||
if self.elapsed_transition_chunks >= self.transition_chunks:
|
||||
self.source_prompt_embeds = self.target_prompt_embeds
|
||||
self.elapsed_transition_chunks = self.transition_chunks
|
||||
|
||||
def _current_alpha(self) -> float:
|
||||
if self.transition_chunks <= 0:
|
||||
return 1.0
|
||||
return min(1.0, self.elapsed_transition_chunks / self.transition_chunks)
|
||||
|
||||
|
||||
class _PendingPromptUpdate(TypedDict):
|
||||
event_id: str
|
||||
prompt: str
|
||||
target_prompt_embeds: torch.Tensor
|
||||
transition_chunks: int
|
||||
|
||||
|
||||
class _PromptUpdateChunkMetadata(TypedDict):
|
||||
started_event_ids: list[str]
|
||||
active_event_ids: list[str]
|
||||
completed_event_ids: list[str]
|
||||
@@ -34,7 +34,7 @@ from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniInteractionPrompt, OmniPromptType
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_MISSING_RPC_RESULT = object()
|
||||
@@ -395,6 +395,19 @@ class StageDiffusionClient(StageClientBase):
|
||||
)
|
||||
)
|
||||
|
||||
async def submit_interaction_async(
|
||||
self,
|
||||
request_id: str,
|
||||
interaction: OmniInteractionPrompt,
|
||||
timeout: float | None = None,
|
||||
) -> Any:
|
||||
"""Apply a midway interaction to an active streaming request."""
|
||||
return await self.collective_rpc_async(
|
||||
"submit_interaction",
|
||||
timeout=timeout,
|
||||
args=(request_id, interaction),
|
||||
)
|
||||
|
||||
async def collective_rpc_async(
|
||||
self,
|
||||
method: str,
|
||||
|
||||
@@ -14,7 +14,7 @@ import copy
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
from torch.profiler import record_function
|
||||
@@ -32,7 +32,7 @@ from vllm_omni.diffusion.compile import regionally_compile
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.forward_context import set_forward_context
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import supports_step_execution
|
||||
from vllm_omni.diffusion.models.interface import SupportsPromptUpdate, supports_prompt_update, supports_step_execution
|
||||
from vllm_omni.diffusion.offloader import get_offload_backend
|
||||
from vllm_omni.diffusion.registry import _NO_CACHE_ACCELERATION
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
@@ -53,6 +53,9 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
from vllm_omni.worker.omni_connector_model_runner_mixin import OmniConnectorModelRunnerMixin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.inputs.data import OmniInteractionPrompt
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -803,3 +806,48 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
self._update_states_after(states, input_batch, pipeline_interrupted)
|
||||
|
||||
return BatchRunnerOutput.from_list(runner_output_list)
|
||||
|
||||
def submit_interaction(
|
||||
self,
|
||||
request_id: str,
|
||||
interaction: OmniInteractionPrompt,
|
||||
) -> None:
|
||||
"""Route a midway interaction to the matching active stepwise request feature."""
|
||||
assert self.pipeline is not None, "Model not loaded. Call load_model() first."
|
||||
if not self.od_config.streaming_output:
|
||||
raise ValueError("submit_interaction requires streaming_output=True")
|
||||
if not self._supports_step_mode():
|
||||
raise ValueError("submit_interaction requires step execution support")
|
||||
|
||||
event = interaction.get("event")
|
||||
if isinstance(event, dict) and "prompt" in event and "multi_modal_data" not in event:
|
||||
# Is a prompt update interaction.
|
||||
self._submit_prompt_update_interaction(request_id, interaction)
|
||||
return
|
||||
|
||||
raise NotImplementedError(
|
||||
"Only text-only prompt update interactions with 'event.prompt' and optional "
|
||||
"'transition_chunks' are supported in this release"
|
||||
)
|
||||
|
||||
def _submit_prompt_update_interaction(
|
||||
self,
|
||||
request_id: str,
|
||||
interaction: OmniInteractionPrompt,
|
||||
) -> None:
|
||||
"""Queue a prompt-update interaction for an active stepwise request."""
|
||||
if not supports_prompt_update(self.pipeline):
|
||||
raise ValueError(f"prompt_update is not supported by pipeline {self.od_config.model_class_name!r}")
|
||||
|
||||
state = self.state_cache.get(request_id)
|
||||
if state is None:
|
||||
raise ValueError(f"No active request state for prompt_update: {request_id!r}")
|
||||
|
||||
event = cast(dict[str, Any], interaction.get("event"))
|
||||
prompt = event["prompt"]
|
||||
transition_chunks = interaction.get("transition_chunks")
|
||||
event_id = interaction.get("event_id")
|
||||
if not isinstance(event_id, str) or not event_id:
|
||||
raise ValueError("event_id must be non-empty")
|
||||
pipeline = cast(SupportsPromptUpdate, self.pipeline)
|
||||
pipeline.prepare_prompt_update(state, prompt, event_id, transition_chunks)
|
||||
@@ -50,6 +50,7 @@ from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput, KVPref
|
||||
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
from vllm_omni.diffusion.worker.utils import BaseRunnerOutput, BatchRunnerOutput
|
||||
from vllm_omni.engine.stage_init_utils import set_death_signal
|
||||
from vllm_omni.inputs.data import OmniInteractionPrompt
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
from vllm_omni.profiler import OmniTorchProfilerWrapper, create_omni_profiler
|
||||
@@ -511,6 +512,15 @@ class DiffusionWorker:
|
||||
# for the frontend server yet.
|
||||
return self.lora_manager.add_adapter(lora_request)
|
||||
|
||||
def submit_interaction(
|
||||
self,
|
||||
request_id: str,
|
||||
interaction: OmniInteractionPrompt,
|
||||
) -> None:
|
||||
"""Apply a midway interaction to an active stepwise request."""
|
||||
assert self.model_runner is not None, "Model runner not initialized"
|
||||
self.model_runner.submit_interaction(request_id, interaction)
|
||||
|
||||
def list_loras(self) -> list[int]:
|
||||
return self.lora_manager.list_adapters()
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ from dataclasses import dataclass, field
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.prompt_update import prompt_update_versions
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
|
||||
@@ -535,12 +536,17 @@ def _same_composition(
|
||||
cached_batch: InputBatch | None,
|
||||
request_ids: list[str],
|
||||
idx_mapping_np: np.ndarray,
|
||||
states: Sequence[StepRequestState],
|
||||
) -> bool:
|
||||
if cached_batch is None:
|
||||
return False
|
||||
if cached_batch.request_ids != request_ids:
|
||||
return False
|
||||
return np.array_equal(cached_batch.idx_mapping_np, idx_mapping_np)
|
||||
if not np.array_equal(cached_batch.idx_mapping_np, idx_mapping_np):
|
||||
return False
|
||||
# Midway prompt updates (typically for video generation) can change prompt_embeds without changing ids/mapping.
|
||||
# In this case, each request state manages the embedding's "version". Use it to determine if cache is still valid.
|
||||
return cached_batch._prompt_update_versions == prompt_update_versions(states)
|
||||
|
||||
|
||||
def _scatter_batch_tensor_by_mapping(
|
||||
@@ -615,6 +621,10 @@ class InputBatch:
|
||||
negative_txt_seq_lens: list[int] | None = None
|
||||
states: Sequence[StepRequestState] = field(default_factory=tuple)
|
||||
|
||||
# For midway prompt updates (typically for video generation) that changes embeddings without changing ids,
|
||||
# Keep a snapshot of the current per-request versions of prompt embeddings for later runtime comparison
|
||||
_prompt_update_versions: tuple[int, ...] = ()
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if len(self.request_ids) != int(self.idx_mapping.numel()):
|
||||
raise ValueError("`request_ids` and `idx_mapping` must have the same length.")
|
||||
@@ -653,6 +663,7 @@ class InputBatch:
|
||||
self.img_shapes = _prepare_img_shapes(states)
|
||||
self.txt_seq_lens = _prepare_seq_lens(states, "txt_seq_lens")
|
||||
self.negative_txt_seq_lens = _prepare_seq_lens(states, "negative_txt_seq_lens")
|
||||
self._prompt_update_versions = prompt_update_versions(states)
|
||||
|
||||
def _repack_dynamic_fields(
|
||||
self,
|
||||
@@ -693,7 +704,7 @@ class InputBatch:
|
||||
selected_states, idx_mapping, idx_mapping_np = _select_states(states, idx_mapping)
|
||||
request_ids = _prepare_request_ids(selected_states)
|
||||
|
||||
if _same_composition(cached_batch, request_ids, idx_mapping_np):
|
||||
if _same_composition(cached_batch, request_ids, idx_mapping_np, selected_states):
|
||||
assert cached_batch is not None
|
||||
cached_batch._repack_dynamic_fields(selected_states)
|
||||
return cached_batch
|
||||
@@ -733,6 +744,7 @@ class InputBatch:
|
||||
"negative_txt_seq_lens",
|
||||
),
|
||||
states=tuple(selected_states),
|
||||
_prompt_update_versions=prompt_update_versions(selected_states),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -48,6 +48,9 @@ def split_diffusion_output_by_request(
|
||||
peak_memory_mb=result.peak_memory_mb,
|
||||
chunk_index=result.chunk_index,
|
||||
total_chunks=result.total_chunks,
|
||||
started_event_ids=result.started_event_ids,
|
||||
active_event_ids=result.active_event_ids,
|
||||
completed_event_ids=result.completed_event_ids,
|
||||
)
|
||||
for idx in range(req.num_reqs)
|
||||
]
|
||||
|
||||
Reference in new issue
Block a user