[Core][Diffusion] Refactor prompt updates into a modality-ready interaction framework (#6294)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
Zeyu Huang | 黃澤宇 authored and GitHub committed 2026-09-07 11:40:04 +08:00
1 parent eeee257913
commit 2df95de421
18 files changed
+1079 -362

No files matched your search

@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,124 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Interaction coordinator to be co-owned by a pipeline and a runner."""
from __future__ import annotations
from typing import TYPE_CHECKING
from vllm_omni.diffusion.interaction.registry import STRUCTURED_HANDLER_REGISTRY
from vllm_omni.diffusion.interaction.types import (
InteractionChunkMetadata,
InteractionPayload,
merge_interaction_metadata,
)
from vllm_omni.diffusion.models.interface import supports_interaction_apply
from vllm_omni.diffusion.worker.utils import StepRequestState
if TYPE_CHECKING:
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
class InteractionCoordinator:
"""Handles mid-way interaction inputs for one loaded pipeline.
Request-local session state lives on ``StepRequestState.interaction_sessions``;
this object is request-agnostic.
"""
def __init__(
self,
handlers: dict[str, InteractionHandler],
*,
model_class_name: str | None = None,
) -> None:
self._handlers = handlers
self.model_class_name = model_class_name
@classmethod
def build(cls, pipeline: object, od_config: OmniDiffusionConfig) -> InteractionCoordinator:
model_class_name = od_config.model_class_name
handlers: dict[str, InteractionHandler] = {}
if supports_interaction_apply(pipeline):
for modality, handler_cls in STRUCTURED_HANDLER_REGISTRY.get(model_class_name or "", {}).items():
handlers[modality] = handler_cls.from_pipeline(pipeline)
return cls(handlers, model_class_name=model_class_name)
def has_modality(self, modality: str) -> bool:
return modality in self._handlers
@property
def needs_chunk_media(self) -> bool:
"""Whether any registered handler needs chunk ``num_frames``/``fps``."""
return any(handler.needs_chunk_media for handler in self._handlers.values())
def get_handler(self, modality: str) -> InteractionHandler:
handler = self._handlers.get(modality)
if handler is None:
raise ValueError(
f"interaction modality {modality!r} is not supported by pipeline {self.model_class_name!r}"
)
return handler
def enqueue(
self,
state: StepRequestState,
*,
modality: str,
event_id: str,
received_at: float,
payload: InteractionPayload,
transition_chunks: int | None,
) -> None:
"""Fan out enqueue to handlers in stable order (prompt first)."""
handler = self.get_handler(modality)
handler.enqueue(
state,
event_id=event_id,
received_at=received_at,
payload=payload,
transition_chunks=transition_chunks,
)
def apply_at_chunk_boundary(
self,
state: StepRequestState,
*,
boundary_at: float,
chunk_index: int | None = None,
num_frames: int | None = None,
fps: float | None = None,
) -> InteractionChunkMetadata:
"""Fan out chunk-boundary apply to handlers in stable order (prompt first)."""
if chunk_index is None:
chunk_index = state.chunk_index
metas: list[InteractionChunkMetadata] = []
for handler in self._handlers_in_apply_order():
if handler.modality not in state.interaction_sessions:
# This function is always called at chunk boundary without knowing if an interaction is really enqueued.
# `state.interaction_sessions[this modality]` is lazy-populated after the first interaction is enqueued.
# So, check here and skip a modality handler if this modality's session is not yet created.
continue
meta = handler.apply_at_chunk_boundary(
state,
boundary_at=boundary_at,
chunk_index=chunk_index,
num_frames=num_frames,
fps=fps,
)
if meta is not None:
metas.append(meta)
return merge_interaction_metadata(metas)
def _handlers_in_apply_order(self) -> list[InteractionHandler]:
ordered: list[InteractionHandler] = []
if "prompt" in self._handlers:
ordered.append(self._handlers["prompt"])
for modality, handler in self._handlers.items():
if modality == "prompt":
continue
ordered.append(handler)
return ordered
+58
View File
@@ -0,0 +1,58 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Interaction-handling mixin for diffusion pipelines. Process interactions at chunk boundaries."""
from __future__ import annotations
import time
from typing import TYPE_CHECKING, cast
from vllm_omni.diffusion.models.interface import SupportsInteractionApply
from vllm_omni.diffusion.worker.utils import StepRequestState
if TYPE_CHECKING:
from vllm_omni.diffusion.interaction.coordinator import InteractionCoordinator
class InteractionMixin:
"""Unified chunk-boundary interaction hook.
Pipelines inherit this mixin and implement ``peek_chunk_media`` to offer playback information about the next chunk.
Diffusion runner calls ``apply_interaction_at_chunk_boundary`` without knowing which modality is being applied.
Per-modality session state lives on ``StepRequestState.interaction_sessions``;
handler strategy objects live on the pipeline/runner ``InteractionCoordinator``.
"""
_interaction_coordinator: InteractionCoordinator | None = None
def apply_interaction_at_chunk_boundary(self, state: StepRequestState) -> None:
"""Advance all active interaction tracks before the next chunk."""
if self._interaction_coordinator is None:
raise RuntimeError(
"interaction coordinator is not initialized; "
"DiffusionModelRunner.load_model must wire InteractionCoordinator "
"onto the pipeline before chunked generation"
)
num_frames: int | None = None
fps: float | None = None
if self._interaction_coordinator.needs_chunk_media:
media = cast(SupportsInteractionApply, self).peek_chunk_media(state)
num_frames = media.num_frames
fps = media.fps
merged = self._interaction_coordinator.apply_at_chunk_boundary(
state,
boundary_at=time.monotonic(),
num_frames=num_frames,
fps=fps,
)
state.interaction_chunk_metadata = merged
def prepare_next_chunk(self, state: StepRequestState) -> None:
"""Set up pipeline state for the next chunk after interaction apply.
Default no-op. Pipelines may override this if needed.
"""
pass
@@ -0,0 +1,62 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Base interaction handler interface."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import ClassVar, Self
from vllm_omni.diffusion.interaction.types import (
InteractionChunkMetadata,
InteractionPayload,
)
from vllm_omni.diffusion.worker.utils import StepRequestState
class InteractionHandler(ABC):
"""Strategy object for one interaction modality.
Handler instances are pipeline/runner-owned and request-agnostic. Per-request
session state lives on ``StepRequestState.interaction_sessions``.
Each concrete modality handler decides how to handle timing from ``received_at``.
"""
modality: ClassVar[str]
# When True, chunk-boundary apply needs ``ChunkMediaSpec`` (num_frames/fps)
# Those information are useful when interaction handler needs interpolation/integration on a frame-by-frame basis
needs_chunk_media: ClassVar[bool] = False
@classmethod
def from_pipeline(cls, pipeline: object) -> Self:
"""Bind a request-agnostic handler to ``pipeline`` when construction needs it.
Default construction ignores the pipeline. Prompt handlers override this to
capture ``encode_prompt`` / device / dtype.
"""
del pipeline
return cls()
@abstractmethod
def enqueue(
self,
state: StepRequestState,
*,
event_id: str,
received_at: float,
payload: InteractionPayload,
transition_chunks: int | None,
) -> None:
"""Validate and queue this track on request-local state."""
@abstractmethod
def apply_at_chunk_boundary(
self,
state: StepRequestState,
*,
boundary_at: float,
chunk_index: int | None = None, # defaults to state.chunk_index when omitted
num_frames: int | None = None, # only set when at least one interaction handler needs it
fps: float | None = None, # only set when at least one interaction handler needs it
) -> InteractionChunkMetadata | None:
"""Advance request-local state and materialize this chunk's effects."""
@@ -0,0 +1,266 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Prompt-track interaction handler (midway prompt updates)."""
from __future__ import annotations
import logging
from collections.abc import Callable, Sequence
from dataclasses import dataclass
from typing import Any, ClassVar
import torch
from typing_extensions import override
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
from vllm_omni.diffusion.interaction.types import (
InteractionChunkMetadata,
InteractionEvent,
InteractionMode,
InteractionPayload,
InteractionSession,
)
from vllm_omni.diffusion.worker.utils import StepRequestState
DEFAULT_TRANSITION_CHUNKS = 3
logger = logging.getLogger(__name__)
@dataclass(kw_only=True)
class QueuedPromptEvent(InteractionEvent):
"""Last-write-wins prompt update waiting for / active at a chunk boundary.
``source_prompt_embeds`` is the lerp start endpoint, staged when this event
is activated (not at enqueue). Live blended embeds live on
``StepRequestState.prompt_embeds``.
"""
mode: InteractionMode = "target"
prompt: str
target_prompt_embeds: torch.Tensor
source_prompt_embeds: torch.Tensor | None = None
def blended_prompt_embeds(self) -> torch.Tensor:
assert self.source_prompt_embeds is not None
alpha = self._current_alpha()
if alpha >= 1.0:
return self.target_prompt_embeds
return (1.0 - alpha) * self.source_prompt_embeds + alpha * self.target_prompt_embeds
def advance_transition(self) -> None:
assert self.source_prompt_embeds is not None
if self.transition_chunks <= 0:
self.source_prompt_embeds = self.target_prompt_embeds
return
self.elapsed_transition_chunks += 1
if self.elapsed_transition_chunks >= self.transition_chunks:
self.source_prompt_embeds = self.target_prompt_embeds
self.elapsed_transition_chunks = float(self.transition_chunks)
def _current_alpha(self) -> float:
if self.transition_chunks <= 0:
return 1.0
return min(1.0, self.elapsed_transition_chunks / self.transition_chunks)
@dataclass
class PromptSession(InteractionSession):
"""Request-local prompt-update session under ``state.interaction_sessions['prompt']``."""
pending_event: QueuedPromptEvent | None = None
active_event: QueuedPromptEvent | None = None
version: int = 0
def prompt_update_versions(states: Sequence[StepRequestState]) -> tuple[int, ...]:
"""Return per-request prompt-update versions for batch cache comparison."""
versions: list[int] = []
for state in states:
session = state.interaction_sessions.get("prompt")
versions.append(session.version if isinstance(session, PromptSession) else 0)
return tuple(versions)
class PromptInteractionHandler(InteractionHandler):
"""Encode-and-queue prompt updates; lerp embeds at chunk boundaries.
Mirrors the former ``PromptUpdateMixin`` behavior. Encoding happens at
enqueue time via an injected ``encode_prompt`` callable.
"""
modality: ClassVar[str] = "prompt"
def __init__(
self,
*,
encode_prompt: Callable[..., tuple[torch.Tensor, ...]],
device: torch.device | str,
dtype: torch.dtype,
) -> None:
self._encode_prompt = encode_prompt
self._device = device
self._dtype = dtype
@classmethod
@override
def from_pipeline(cls, pipeline: Any) -> PromptInteractionHandler:
"""Build a handler from a diffusion pipeline that exposes ``encode_prompt``."""
return cls(
encode_prompt=pipeline.encode_prompt,
device=pipeline.device,
dtype=pipeline.transformer.dtype,
)
@override
def enqueue(
self,
state: StepRequestState,
*,
event_id: str,
received_at: float,
payload: InteractionPayload,
transition_chunks: int | None,
) -> None:
"""Prompt updates are last-write-win and unbuffered at chunk boundary."""
prompt = payload.get("prompt")
if not isinstance(prompt, str) or not prompt:
raise ValueError("prompt must be non-empty")
if not event_id:
raise ValueError("event_id must be non-empty")
if state.prompt_embeds is None:
raise ValueError(
f"prompt_update is not allowed before initial generation has started (request_id={state.request_id!r})"
)
duration = DEFAULT_TRANSITION_CHUNKS if transition_chunks is None else transition_chunks
if duration < 0:
raise ValueError("transition_chunks must be >= 0")
target_prompt_embeds, _ = self._encode_prompt(
prompt=prompt,
negative_prompt=None,
do_classifier_free_guidance=False,
num_videos_per_prompt=state.sampling.num_outputs_per_prompt,
max_sequence_length=state.sampling.max_sequence_length,
device=self._device,
dtype=self._dtype,
)
session = state.interaction_sessions.setdefault("prompt", PromptSession())
assert isinstance(session, PromptSession)
with session.lock:
# Chunk-level LWW: replace any prior pending event.
session.pending_event = QueuedPromptEvent(
event_id=event_id,
received_at=received_at,
transition_chunks=duration,
prompt=prompt,
target_prompt_embeds=target_prompt_embeds,
)
@override
def apply_at_chunk_boundary(
self,
state: StepRequestState,
*,
boundary_at: float,
chunk_index: int | None = None,
num_frames: int | None = None,
fps: float | None = None,
) -> InteractionChunkMetadata | None:
"""Advance or start prompt interpolation before the next chunk."""
# Prompt lerp is chunk-LWW; media timeline and caller chunk_index unused.
del chunk_index, num_frames, fps
session = state.interaction_sessions.get("prompt")
assert isinstance(session, PromptSession)
embeds_changed = False
next_chunk_index = state.chunk_index
started_event_ids: list[str] = []
active_event_ids: list[str] = []
completed_event_ids: list[str] = []
with session.lock:
# Prompt ignores frame scheduling but still records the boundary clock.
session.last_boundary_at = boundary_at
pending_event = session.pending_event
session.pending_event = None
active_event = session.active_event
# If current transition is not complete, advance it.
# After completion, leave active_event in place (so a later pending
# update can abort/overwrite), but do not keep bumping the version.
if active_event is not None:
in_transition = (
active_event.transition_chunks > 0
and active_event.elapsed_transition_chunks < active_event.transition_chunks
)
if in_transition:
active_event.advance_transition()
state.prompt_embeds = active_event.blended_prompt_embeds()
embeds_changed = True
active_event_ids.append(active_event.event_id)
if active_event.elapsed_transition_chunks >= active_event.transition_chunks:
state.prompt_embeds = active_event.target_prompt_embeds
active_event.source_prompt_embeds = active_event.target_prompt_embeds
completed_event_ids.append(active_event.event_id)
logger.debug(
"prompt_update transition complete request_id=%s next_chunk_index=%d prompt=%.20s...",
state.request_id,
next_chunk_index,
active_event.prompt,
)
# If a new prompt update is pending, start a new transition.
if pending_event is not None:
if state.prompt_embeds is None:
raise RuntimeError(
"internal error: trying to apply a pending prompt update but "
f"current prompt_embeds is None (request_id={state.request_id!r})"
)
# Stage lerp-start embeds at activation (after any same-boundary advance).
pending_event.source_prompt_embeds = state.prompt_embeds.detach().clone()
pending_event.elapsed_transition_chunks = 0.0
session.active_event = pending_event
active_event = pending_event
event_id = pending_event.event_id
duration = pending_event.transition_chunks
prompt = pending_event.prompt
target = pending_event.target_prompt_embeds
started_event_ids.append(event_id)
active_event_ids.append(event_id)
if duration <= 0:
# A hard/immediate transition
state.prompt_embeds = target
pending_event.source_prompt_embeds = target
completed_event_ids.append(event_id)
logger.debug(
"prompt_update sharp transition request_id=%s next_chunk_index=%d prompt=%.20s...",
state.request_id,
next_chunk_index,
prompt,
)
else:
# A transition that really takes some time
active_event.advance_transition()
state.prompt_embeds = active_event.blended_prompt_embeds()
if active_event.elapsed_transition_chunks >= active_event.transition_chunks:
state.prompt_embeds = active_event.target_prompt_embeds
active_event.source_prompt_embeds = active_event.target_prompt_embeds
completed_event_ids.append(event_id)
logger.debug(
"prompt_update transition start request_id=%s next_chunk_index=%d prompt=%.20s...",
state.request_id,
next_chunk_index,
prompt,
)
embeds_changed = True
# Indicate that prompt embeddings have changed---clear input batch cache.
if embeds_changed:
session.version += 1
return InteractionChunkMetadata(
started_event_ids=started_event_ids,
active_event_ids=active_event_ids,
completed_event_ids=completed_event_ids,
)
@@ -0,0 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Registry of mid-way interaction handlers during a diffusion generation.
A pipeline only declares the modalities that it supports.
And a pipeline can choose which handler for each modality (there can be more than one handler for a modality).
Outer key: pipeline architecture name (``od_config.model_class_name``).
Inner key: modality name (e.g. ``\"prompt\"``).
"""
from __future__ import annotations
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
from vllm_omni.diffusion.interaction.modality_handlers.prompt import PromptInteractionHandler
STRUCTURED_HANDLER_REGISTRY: dict[str, dict[str, type[InteractionHandler]]] = {
# Pipeline class name -> modality -> handler class.
"HeliosPipeline": {
"prompt": PromptInteractionHandler,
},
"HeliosPyramidPipeline": {
"prompt": PromptInteractionHandler,
},
}
+112
View File
@@ -0,0 +1,112 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared types for midway interaction handlers."""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass, field
from threading import Lock
from typing import Literal
# Wire payload for one modality track. Handlers validate keys they need.
InteractionPayload = Mapping[str, object]
# Shared command semantics across modalities that support them.
# Prompt specialization uses ``"target"`` only.
InteractionMode = Literal["target"]
@dataclass(frozen=True)
class ChunkMediaSpec:
"""Media extent of one generation chunk, for interaction timeline mapping."""
num_frames: int
fps: float
def __post_init__(self) -> None:
if self.num_frames <= 0:
raise ValueError(f"ChunkMediaSpec.num_frames must be > 0, got {self.num_frames}")
if self.fps <= 0:
raise ValueError(f"ChunkMediaSpec.fps must be > 0, got {self.fps}")
@property
def duration_s(self) -> float:
return float(self.num_frames) / float(self.fps)
@dataclass(kw_only=True)
class InteractionEvent:
"""Shared envelope for one queued or in-flight interaction command. **Intended to be subclassed**.
Example:
- ``QueuedPromptEvent`` in ``vllm_omni.diffusion.interaction.modality_handlers.prompt``
"""
event_id: str
received_at: float
mode: InteractionMode
transition_chunks: int
elapsed_transition_chunks: float = 0.0
@dataclass
class InteractionSession:
"""Request-local session base for one interaction modality. **Intended to be subclassed**.
Concrete modalities subclass this and store their own pending/active state.
``last_boundary_at`` is available for frame-scheduled modalities; chunk-LWW
modalities may ignore it.
Example:
- ``PromptSession`` in ``vllm_omni.diffusion.interaction.modality_handlers.prompt``
"""
lock: Lock = field(default_factory=Lock, repr=False)
last_boundary_at: float | None = None
@dataclass
class InteractionChunkMetadata:
"""Per-modality (or merged) event-id bookkeeping for one chunk boundary."""
started_event_ids: list[str] = field(default_factory=list)
active_event_ids: list[str] = field(default_factory=list)
completed_event_ids: list[str] = field(default_factory=list)
def as_dict(self) -> dict[str, list[str]]:
return {
"started_event_ids": list(self.started_event_ids),
"active_event_ids": list(self.active_event_ids),
"completed_event_ids": list(self.completed_event_ids),
}
def merge_interaction_metadata(
metas: list[InteractionChunkMetadata],
) -> InteractionChunkMetadata:
"""Merge per-handler metadata while preserving order and de-duplicating ids."""
started: list[str] = []
active: list[str] = []
completed: list[str] = []
seen_started: set[str] = set()
seen_active: set[str] = set()
seen_completed: set[str] = set()
for meta in metas:
for event_id in meta.started_event_ids:
if event_id not in seen_started:
seen_started.add(event_id)
started.append(event_id)
for event_id in meta.active_event_ids:
if event_id not in seen_active:
seen_active.add(event_id)
active.append(event_id)
for event_id in meta.completed_event_ids:
if event_id not in seen_completed:
seen_completed.add(event_id)
completed.append(event_id)
return InteractionChunkMetadata(
started_event_ids=started,
active_event_ids=active,
completed_event_ids=completed,
)