[Core][Diffusion] Refactor prompt updates into a modality-ready interaction framework (#6294)
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
1 parent
eeee257913
commit
2df95de421
18 files changed
+1079
-362
No files matched your search
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,124 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Interaction coordinator to be co-owned by a pipeline and a runner."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm_omni.diffusion.interaction.registry import STRUCTURED_HANDLER_REGISTRY
|
||||
from vllm_omni.diffusion.interaction.types import (
|
||||
InteractionChunkMetadata,
|
||||
InteractionPayload,
|
||||
merge_interaction_metadata,
|
||||
)
|
||||
from vllm_omni.diffusion.models.interface import supports_interaction_apply
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
|
||||
|
||||
|
||||
class InteractionCoordinator:
|
||||
"""Handles mid-way interaction inputs for one loaded pipeline.
|
||||
|
||||
Request-local session state lives on ``StepRequestState.interaction_sessions``;
|
||||
this object is request-agnostic.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
handlers: dict[str, InteractionHandler],
|
||||
*,
|
||||
model_class_name: str | None = None,
|
||||
) -> None:
|
||||
self._handlers = handlers
|
||||
self.model_class_name = model_class_name
|
||||
|
||||
@classmethod
|
||||
def build(cls, pipeline: object, od_config: OmniDiffusionConfig) -> InteractionCoordinator:
|
||||
model_class_name = od_config.model_class_name
|
||||
handlers: dict[str, InteractionHandler] = {}
|
||||
|
||||
if supports_interaction_apply(pipeline):
|
||||
for modality, handler_cls in STRUCTURED_HANDLER_REGISTRY.get(model_class_name or "", {}).items():
|
||||
handlers[modality] = handler_cls.from_pipeline(pipeline)
|
||||
|
||||
return cls(handlers, model_class_name=model_class_name)
|
||||
|
||||
def has_modality(self, modality: str) -> bool:
|
||||
return modality in self._handlers
|
||||
|
||||
@property
|
||||
def needs_chunk_media(self) -> bool:
|
||||
"""Whether any registered handler needs chunk ``num_frames``/``fps``."""
|
||||
return any(handler.needs_chunk_media for handler in self._handlers.values())
|
||||
|
||||
def get_handler(self, modality: str) -> InteractionHandler:
|
||||
handler = self._handlers.get(modality)
|
||||
if handler is None:
|
||||
raise ValueError(
|
||||
f"interaction modality {modality!r} is not supported by pipeline {self.model_class_name!r}"
|
||||
)
|
||||
return handler
|
||||
|
||||
def enqueue(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
modality: str,
|
||||
event_id: str,
|
||||
received_at: float,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
"""Fan out enqueue to handlers in stable order (prompt first)."""
|
||||
handler = self.get_handler(modality)
|
||||
handler.enqueue(
|
||||
state,
|
||||
event_id=event_id,
|
||||
received_at=received_at,
|
||||
payload=payload,
|
||||
transition_chunks=transition_chunks,
|
||||
)
|
||||
|
||||
def apply_at_chunk_boundary(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
boundary_at: float,
|
||||
chunk_index: int | None = None,
|
||||
num_frames: int | None = None,
|
||||
fps: float | None = None,
|
||||
) -> InteractionChunkMetadata:
|
||||
"""Fan out chunk-boundary apply to handlers in stable order (prompt first)."""
|
||||
if chunk_index is None:
|
||||
chunk_index = state.chunk_index
|
||||
metas: list[InteractionChunkMetadata] = []
|
||||
for handler in self._handlers_in_apply_order():
|
||||
if handler.modality not in state.interaction_sessions:
|
||||
# This function is always called at chunk boundary without knowing if an interaction is really enqueued.
|
||||
# `state.interaction_sessions[this modality]` is lazy-populated after the first interaction is enqueued.
|
||||
# So, check here and skip a modality handler if this modality's session is not yet created.
|
||||
continue
|
||||
meta = handler.apply_at_chunk_boundary(
|
||||
state,
|
||||
boundary_at=boundary_at,
|
||||
chunk_index=chunk_index,
|
||||
num_frames=num_frames,
|
||||
fps=fps,
|
||||
)
|
||||
if meta is not None:
|
||||
metas.append(meta)
|
||||
return merge_interaction_metadata(metas)
|
||||
|
||||
def _handlers_in_apply_order(self) -> list[InteractionHandler]:
|
||||
ordered: list[InteractionHandler] = []
|
||||
if "prompt" in self._handlers:
|
||||
ordered.append(self._handlers["prompt"])
|
||||
for modality, handler in self._handlers.items():
|
||||
if modality == "prompt":
|
||||
continue
|
||||
ordered.append(handler)
|
||||
return ordered
|
||||
@@ -0,0 +1,58 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Interaction-handling mixin for diffusion pipelines. Process interactions at chunk boundaries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from vllm_omni.diffusion.models.interface import SupportsInteractionApply
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.interaction.coordinator import InteractionCoordinator
|
||||
|
||||
|
||||
class InteractionMixin:
|
||||
"""Unified chunk-boundary interaction hook.
|
||||
|
||||
Pipelines inherit this mixin and implement ``peek_chunk_media`` to offer playback information about the next chunk.
|
||||
Diffusion runner calls ``apply_interaction_at_chunk_boundary`` without knowing which modality is being applied.
|
||||
|
||||
Per-modality session state lives on ``StepRequestState.interaction_sessions``;
|
||||
handler strategy objects live on the pipeline/runner ``InteractionCoordinator``.
|
||||
"""
|
||||
|
||||
_interaction_coordinator: InteractionCoordinator | None = None
|
||||
|
||||
def apply_interaction_at_chunk_boundary(self, state: StepRequestState) -> None:
|
||||
"""Advance all active interaction tracks before the next chunk."""
|
||||
if self._interaction_coordinator is None:
|
||||
raise RuntimeError(
|
||||
"interaction coordinator is not initialized; "
|
||||
"DiffusionModelRunner.load_model must wire InteractionCoordinator "
|
||||
"onto the pipeline before chunked generation"
|
||||
)
|
||||
|
||||
num_frames: int | None = None
|
||||
fps: float | None = None
|
||||
if self._interaction_coordinator.needs_chunk_media:
|
||||
media = cast(SupportsInteractionApply, self).peek_chunk_media(state)
|
||||
num_frames = media.num_frames
|
||||
fps = media.fps
|
||||
|
||||
merged = self._interaction_coordinator.apply_at_chunk_boundary(
|
||||
state,
|
||||
boundary_at=time.monotonic(),
|
||||
num_frames=num_frames,
|
||||
fps=fps,
|
||||
)
|
||||
state.interaction_chunk_metadata = merged
|
||||
|
||||
def prepare_next_chunk(self, state: StepRequestState) -> None:
|
||||
"""Set up pipeline state for the next chunk after interaction apply.
|
||||
|
||||
Default no-op. Pipelines may override this if needed.
|
||||
"""
|
||||
pass
|
||||
@@ -0,0 +1,62 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Base interaction handler interface."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import ClassVar, Self
|
||||
|
||||
from vllm_omni.diffusion.interaction.types import (
|
||||
InteractionChunkMetadata,
|
||||
InteractionPayload,
|
||||
)
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
|
||||
class InteractionHandler(ABC):
|
||||
"""Strategy object for one interaction modality.
|
||||
|
||||
Handler instances are pipeline/runner-owned and request-agnostic. Per-request
|
||||
session state lives on ``StepRequestState.interaction_sessions``.
|
||||
Each concrete modality handler decides how to handle timing from ``received_at``.
|
||||
"""
|
||||
|
||||
modality: ClassVar[str]
|
||||
# When True, chunk-boundary apply needs ``ChunkMediaSpec`` (num_frames/fps)
|
||||
# Those information are useful when interaction handler needs interpolation/integration on a frame-by-frame basis
|
||||
needs_chunk_media: ClassVar[bool] = False
|
||||
|
||||
@classmethod
|
||||
def from_pipeline(cls, pipeline: object) -> Self:
|
||||
"""Bind a request-agnostic handler to ``pipeline`` when construction needs it.
|
||||
|
||||
Default construction ignores the pipeline. Prompt handlers override this to
|
||||
capture ``encode_prompt`` / device / dtype.
|
||||
"""
|
||||
del pipeline
|
||||
return cls()
|
||||
|
||||
@abstractmethod
|
||||
def enqueue(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
event_id: str,
|
||||
received_at: float,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
"""Validate and queue this track on request-local state."""
|
||||
|
||||
@abstractmethod
|
||||
def apply_at_chunk_boundary(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
boundary_at: float,
|
||||
chunk_index: int | None = None, # defaults to state.chunk_index when omitted
|
||||
num_frames: int | None = None, # only set when at least one interaction handler needs it
|
||||
fps: float | None = None, # only set when at least one interaction handler needs it
|
||||
) -> InteractionChunkMetadata | None:
|
||||
"""Advance request-local state and materialize this chunk's effects."""
|
||||
@@ -0,0 +1,266 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Prompt-track interaction handler (midway prompt updates)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import torch
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
|
||||
from vllm_omni.diffusion.interaction.types import (
|
||||
InteractionChunkMetadata,
|
||||
InteractionEvent,
|
||||
InteractionMode,
|
||||
InteractionPayload,
|
||||
InteractionSession,
|
||||
)
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
DEFAULT_TRANSITION_CHUNKS = 3
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class QueuedPromptEvent(InteractionEvent):
|
||||
"""Last-write-wins prompt update waiting for / active at a chunk boundary.
|
||||
|
||||
``source_prompt_embeds`` is the lerp start endpoint, staged when this event
|
||||
is activated (not at enqueue). Live blended embeds live on
|
||||
``StepRequestState.prompt_embeds``.
|
||||
"""
|
||||
|
||||
mode: InteractionMode = "target"
|
||||
prompt: str
|
||||
target_prompt_embeds: torch.Tensor
|
||||
source_prompt_embeds: torch.Tensor | None = None
|
||||
|
||||
def blended_prompt_embeds(self) -> torch.Tensor:
|
||||
assert self.source_prompt_embeds is not None
|
||||
alpha = self._current_alpha()
|
||||
if alpha >= 1.0:
|
||||
return self.target_prompt_embeds
|
||||
return (1.0 - alpha) * self.source_prompt_embeds + alpha * self.target_prompt_embeds
|
||||
|
||||
def advance_transition(self) -> None:
|
||||
assert self.source_prompt_embeds is not None
|
||||
if self.transition_chunks <= 0:
|
||||
self.source_prompt_embeds = self.target_prompt_embeds
|
||||
return
|
||||
self.elapsed_transition_chunks += 1
|
||||
if self.elapsed_transition_chunks >= self.transition_chunks:
|
||||
self.source_prompt_embeds = self.target_prompt_embeds
|
||||
self.elapsed_transition_chunks = float(self.transition_chunks)
|
||||
|
||||
def _current_alpha(self) -> float:
|
||||
if self.transition_chunks <= 0:
|
||||
return 1.0
|
||||
return min(1.0, self.elapsed_transition_chunks / self.transition_chunks)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PromptSession(InteractionSession):
|
||||
"""Request-local prompt-update session under ``state.interaction_sessions['prompt']``."""
|
||||
|
||||
pending_event: QueuedPromptEvent | None = None
|
||||
active_event: QueuedPromptEvent | None = None
|
||||
version: int = 0
|
||||
|
||||
|
||||
def prompt_update_versions(states: Sequence[StepRequestState]) -> tuple[int, ...]:
|
||||
"""Return per-request prompt-update versions for batch cache comparison."""
|
||||
versions: list[int] = []
|
||||
for state in states:
|
||||
session = state.interaction_sessions.get("prompt")
|
||||
versions.append(session.version if isinstance(session, PromptSession) else 0)
|
||||
return tuple(versions)
|
||||
|
||||
|
||||
class PromptInteractionHandler(InteractionHandler):
|
||||
"""Encode-and-queue prompt updates; lerp embeds at chunk boundaries.
|
||||
|
||||
Mirrors the former ``PromptUpdateMixin`` behavior. Encoding happens at
|
||||
enqueue time via an injected ``encode_prompt`` callable.
|
||||
"""
|
||||
|
||||
modality: ClassVar[str] = "prompt"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
encode_prompt: Callable[..., tuple[torch.Tensor, ...]],
|
||||
device: torch.device | str,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
self._encode_prompt = encode_prompt
|
||||
self._device = device
|
||||
self._dtype = dtype
|
||||
|
||||
@classmethod
|
||||
@override
|
||||
def from_pipeline(cls, pipeline: Any) -> PromptInteractionHandler:
|
||||
"""Build a handler from a diffusion pipeline that exposes ``encode_prompt``."""
|
||||
return cls(
|
||||
encode_prompt=pipeline.encode_prompt,
|
||||
device=pipeline.device,
|
||||
dtype=pipeline.transformer.dtype,
|
||||
)
|
||||
|
||||
@override
|
||||
def enqueue(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
event_id: str,
|
||||
received_at: float,
|
||||
payload: InteractionPayload,
|
||||
transition_chunks: int | None,
|
||||
) -> None:
|
||||
"""Prompt updates are last-write-win and unbuffered at chunk boundary."""
|
||||
prompt = payload.get("prompt")
|
||||
if not isinstance(prompt, str) or not prompt:
|
||||
raise ValueError("prompt must be non-empty")
|
||||
if not event_id:
|
||||
raise ValueError("event_id must be non-empty")
|
||||
if state.prompt_embeds is None:
|
||||
raise ValueError(
|
||||
f"prompt_update is not allowed before initial generation has started (request_id={state.request_id!r})"
|
||||
)
|
||||
duration = DEFAULT_TRANSITION_CHUNKS if transition_chunks is None else transition_chunks
|
||||
if duration < 0:
|
||||
raise ValueError("transition_chunks must be >= 0")
|
||||
|
||||
target_prompt_embeds, _ = self._encode_prompt(
|
||||
prompt=prompt,
|
||||
negative_prompt=None,
|
||||
do_classifier_free_guidance=False,
|
||||
num_videos_per_prompt=state.sampling.num_outputs_per_prompt,
|
||||
max_sequence_length=state.sampling.max_sequence_length,
|
||||
device=self._device,
|
||||
dtype=self._dtype,
|
||||
)
|
||||
session = state.interaction_sessions.setdefault("prompt", PromptSession())
|
||||
assert isinstance(session, PromptSession)
|
||||
with session.lock:
|
||||
# Chunk-level LWW: replace any prior pending event.
|
||||
session.pending_event = QueuedPromptEvent(
|
||||
event_id=event_id,
|
||||
received_at=received_at,
|
||||
transition_chunks=duration,
|
||||
prompt=prompt,
|
||||
target_prompt_embeds=target_prompt_embeds,
|
||||
)
|
||||
|
||||
@override
|
||||
def apply_at_chunk_boundary(
|
||||
self,
|
||||
state: StepRequestState,
|
||||
*,
|
||||
boundary_at: float,
|
||||
chunk_index: int | None = None,
|
||||
num_frames: int | None = None,
|
||||
fps: float | None = None,
|
||||
) -> InteractionChunkMetadata | None:
|
||||
"""Advance or start prompt interpolation before the next chunk."""
|
||||
# Prompt lerp is chunk-LWW; media timeline and caller chunk_index unused.
|
||||
del chunk_index, num_frames, fps
|
||||
session = state.interaction_sessions.get("prompt")
|
||||
assert isinstance(session, PromptSession)
|
||||
|
||||
embeds_changed = False
|
||||
next_chunk_index = state.chunk_index
|
||||
started_event_ids: list[str] = []
|
||||
active_event_ids: list[str] = []
|
||||
completed_event_ids: list[str] = []
|
||||
|
||||
with session.lock:
|
||||
# Prompt ignores frame scheduling but still records the boundary clock.
|
||||
session.last_boundary_at = boundary_at
|
||||
pending_event = session.pending_event
|
||||
session.pending_event = None
|
||||
active_event = session.active_event
|
||||
|
||||
# If current transition is not complete, advance it.
|
||||
# After completion, leave active_event in place (so a later pending
|
||||
# update can abort/overwrite), but do not keep bumping the version.
|
||||
if active_event is not None:
|
||||
in_transition = (
|
||||
active_event.transition_chunks > 0
|
||||
and active_event.elapsed_transition_chunks < active_event.transition_chunks
|
||||
)
|
||||
if in_transition:
|
||||
active_event.advance_transition()
|
||||
state.prompt_embeds = active_event.blended_prompt_embeds()
|
||||
embeds_changed = True
|
||||
active_event_ids.append(active_event.event_id)
|
||||
if active_event.elapsed_transition_chunks >= active_event.transition_chunks:
|
||||
state.prompt_embeds = active_event.target_prompt_embeds
|
||||
active_event.source_prompt_embeds = active_event.target_prompt_embeds
|
||||
completed_event_ids.append(active_event.event_id)
|
||||
logger.debug(
|
||||
"prompt_update transition complete request_id=%s next_chunk_index=%d prompt=%.20s...",
|
||||
state.request_id,
|
||||
next_chunk_index,
|
||||
active_event.prompt,
|
||||
)
|
||||
|
||||
# If a new prompt update is pending, start a new transition.
|
||||
if pending_event is not None:
|
||||
if state.prompt_embeds is None:
|
||||
raise RuntimeError(
|
||||
"internal error: trying to apply a pending prompt update but "
|
||||
f"current prompt_embeds is None (request_id={state.request_id!r})"
|
||||
)
|
||||
# Stage lerp-start embeds at activation (after any same-boundary advance).
|
||||
pending_event.source_prompt_embeds = state.prompt_embeds.detach().clone()
|
||||
pending_event.elapsed_transition_chunks = 0.0
|
||||
session.active_event = pending_event
|
||||
active_event = pending_event
|
||||
event_id = pending_event.event_id
|
||||
duration = pending_event.transition_chunks
|
||||
prompt = pending_event.prompt
|
||||
target = pending_event.target_prompt_embeds
|
||||
started_event_ids.append(event_id)
|
||||
active_event_ids.append(event_id)
|
||||
if duration <= 0:
|
||||
# A hard/immediate transition
|
||||
state.prompt_embeds = target
|
||||
pending_event.source_prompt_embeds = target
|
||||
completed_event_ids.append(event_id)
|
||||
logger.debug(
|
||||
"prompt_update sharp transition request_id=%s next_chunk_index=%d prompt=%.20s...",
|
||||
state.request_id,
|
||||
next_chunk_index,
|
||||
prompt,
|
||||
)
|
||||
else:
|
||||
# A transition that really takes some time
|
||||
active_event.advance_transition()
|
||||
state.prompt_embeds = active_event.blended_prompt_embeds()
|
||||
if active_event.elapsed_transition_chunks >= active_event.transition_chunks:
|
||||
state.prompt_embeds = active_event.target_prompt_embeds
|
||||
active_event.source_prompt_embeds = active_event.target_prompt_embeds
|
||||
completed_event_ids.append(event_id)
|
||||
logger.debug(
|
||||
"prompt_update transition start request_id=%s next_chunk_index=%d prompt=%.20s...",
|
||||
state.request_id,
|
||||
next_chunk_index,
|
||||
prompt,
|
||||
)
|
||||
embeds_changed = True
|
||||
|
||||
# Indicate that prompt embeddings have changed---clear input batch cache.
|
||||
if embeds_changed:
|
||||
session.version += 1
|
||||
|
||||
return InteractionChunkMetadata(
|
||||
started_event_ids=started_event_ids,
|
||||
active_event_ids=active_event_ids,
|
||||
completed_event_ids=completed_event_ids,
|
||||
)
|
||||
@@ -0,0 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Registry of mid-way interaction handlers during a diffusion generation.
|
||||
A pipeline only declares the modalities that it supports.
|
||||
And a pipeline can choose which handler for each modality (there can be more than one handler for a modality).
|
||||
|
||||
Outer key: pipeline architecture name (``od_config.model_class_name``).
|
||||
Inner key: modality name (e.g. ``\"prompt\"``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.base import InteractionHandler
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.prompt import PromptInteractionHandler
|
||||
|
||||
STRUCTURED_HANDLER_REGISTRY: dict[str, dict[str, type[InteractionHandler]]] = {
|
||||
# Pipeline class name -> modality -> handler class.
|
||||
"HeliosPipeline": {
|
||||
"prompt": PromptInteractionHandler,
|
||||
},
|
||||
"HeliosPyramidPipeline": {
|
||||
"prompt": PromptInteractionHandler,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared types for midway interaction handlers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from threading import Lock
|
||||
from typing import Literal
|
||||
|
||||
# Wire payload for one modality track. Handlers validate keys they need.
|
||||
InteractionPayload = Mapping[str, object]
|
||||
|
||||
# Shared command semantics across modalities that support them.
|
||||
# Prompt specialization uses ``"target"`` only.
|
||||
InteractionMode = Literal["target"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChunkMediaSpec:
|
||||
"""Media extent of one generation chunk, for interaction timeline mapping."""
|
||||
|
||||
num_frames: int
|
||||
fps: float
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.num_frames <= 0:
|
||||
raise ValueError(f"ChunkMediaSpec.num_frames must be > 0, got {self.num_frames}")
|
||||
if self.fps <= 0:
|
||||
raise ValueError(f"ChunkMediaSpec.fps must be > 0, got {self.fps}")
|
||||
|
||||
@property
|
||||
def duration_s(self) -> float:
|
||||
return float(self.num_frames) / float(self.fps)
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class InteractionEvent:
|
||||
"""Shared envelope for one queued or in-flight interaction command. **Intended to be subclassed**.
|
||||
|
||||
Example:
|
||||
- ``QueuedPromptEvent`` in ``vllm_omni.diffusion.interaction.modality_handlers.prompt``
|
||||
"""
|
||||
|
||||
event_id: str
|
||||
received_at: float
|
||||
mode: InteractionMode
|
||||
transition_chunks: int
|
||||
elapsed_transition_chunks: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class InteractionSession:
|
||||
"""Request-local session base for one interaction modality. **Intended to be subclassed**.
|
||||
|
||||
Concrete modalities subclass this and store their own pending/active state.
|
||||
``last_boundary_at`` is available for frame-scheduled modalities; chunk-LWW
|
||||
modalities may ignore it.
|
||||
|
||||
Example:
|
||||
- ``PromptSession`` in ``vllm_omni.diffusion.interaction.modality_handlers.prompt``
|
||||
"""
|
||||
|
||||
lock: Lock = field(default_factory=Lock, repr=False)
|
||||
last_boundary_at: float | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class InteractionChunkMetadata:
|
||||
"""Per-modality (or merged) event-id bookkeeping for one chunk boundary."""
|
||||
|
||||
started_event_ids: list[str] = field(default_factory=list)
|
||||
active_event_ids: list[str] = field(default_factory=list)
|
||||
completed_event_ids: list[str] = field(default_factory=list)
|
||||
|
||||
def as_dict(self) -> dict[str, list[str]]:
|
||||
return {
|
||||
"started_event_ids": list(self.started_event_ids),
|
||||
"active_event_ids": list(self.active_event_ids),
|
||||
"completed_event_ids": list(self.completed_event_ids),
|
||||
}
|
||||
|
||||
|
||||
def merge_interaction_metadata(
|
||||
metas: list[InteractionChunkMetadata],
|
||||
) -> InteractionChunkMetadata:
|
||||
"""Merge per-handler metadata while preserving order and de-duplicating ids."""
|
||||
started: list[str] = []
|
||||
active: list[str] = []
|
||||
completed: list[str] = []
|
||||
seen_started: set[str] = set()
|
||||
seen_active: set[str] = set()
|
||||
seen_completed: set[str] = set()
|
||||
for meta in metas:
|
||||
for event_id in meta.started_event_ids:
|
||||
if event_id not in seen_started:
|
||||
seen_started.add(event_id)
|
||||
started.append(event_id)
|
||||
for event_id in meta.active_event_ids:
|
||||
if event_id not in seen_active:
|
||||
seen_active.add(event_id)
|
||||
active.append(event_id)
|
||||
for event_id in meta.completed_event_ids:
|
||||
if event_id not in seen_completed:
|
||||
seen_completed.add(event_id)
|
||||
completed.append(event_id)
|
||||
return InteractionChunkMetadata(
|
||||
started_event_ids=started,
|
||||
active_event_ids=active,
|
||||
completed_event_ids=completed,
|
||||
)
|
||||
Reference in new issue
Block a user