[Frontend] Add midway prompt update for streaming video generation (#4652)
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
1 parent
1f01d05603
commit
eaf90d8eff
28 files changed
+2615
-133
No files matched your search
@@ -14,7 +14,7 @@ import copy
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
from torch.profiler import record_function
|
||||
@@ -32,7 +32,7 @@ from vllm_omni.diffusion.compile import regionally_compile
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.forward_context import set_forward_context
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import supports_step_execution
|
||||
from vllm_omni.diffusion.models.interface import SupportsPromptUpdate, supports_prompt_update, supports_step_execution
|
||||
from vllm_omni.diffusion.offloader import get_offload_backend
|
||||
from vllm_omni.diffusion.registry import _NO_CACHE_ACCELERATION
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
@@ -53,6 +53,9 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
from vllm_omni.worker.omni_connector_model_runner_mixin import OmniConnectorModelRunnerMixin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.inputs.data import OmniInteractionPrompt
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -803,3 +806,48 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
self._update_states_after(states, input_batch, pipeline_interrupted)
|
||||
|
||||
return BatchRunnerOutput.from_list(runner_output_list)
|
||||
|
||||
def submit_interaction(
|
||||
self,
|
||||
request_id: str,
|
||||
interaction: OmniInteractionPrompt,
|
||||
) -> None:
|
||||
"""Route a midway interaction to the matching active stepwise request feature."""
|
||||
assert self.pipeline is not None, "Model not loaded. Call load_model() first."
|
||||
if not self.od_config.streaming_output:
|
||||
raise ValueError("submit_interaction requires streaming_output=True")
|
||||
if not self._supports_step_mode():
|
||||
raise ValueError("submit_interaction requires step execution support")
|
||||
|
||||
event = interaction.get("event")
|
||||
if isinstance(event, dict) and "prompt" in event and "multi_modal_data" not in event:
|
||||
# Is a prompt update interaction.
|
||||
self._submit_prompt_update_interaction(request_id, interaction)
|
||||
return
|
||||
|
||||
raise NotImplementedError(
|
||||
"Only text-only prompt update interactions with 'event.prompt' and optional "
|
||||
"'transition_chunks' are supported in this release"
|
||||
)
|
||||
|
||||
def _submit_prompt_update_interaction(
|
||||
self,
|
||||
request_id: str,
|
||||
interaction: OmniInteractionPrompt,
|
||||
) -> None:
|
||||
"""Queue a prompt-update interaction for an active stepwise request."""
|
||||
if not supports_prompt_update(self.pipeline):
|
||||
raise ValueError(f"prompt_update is not supported by pipeline {self.od_config.model_class_name!r}")
|
||||
|
||||
state = self.state_cache.get(request_id)
|
||||
if state is None:
|
||||
raise ValueError(f"No active request state for prompt_update: {request_id!r}")
|
||||
|
||||
event = cast(dict[str, Any], interaction.get("event"))
|
||||
prompt = event["prompt"]
|
||||
transition_chunks = interaction.get("transition_chunks")
|
||||
event_id = interaction.get("event_id")
|
||||
if not isinstance(event_id, str) or not event_id:
|
||||
raise ValueError("event_id must be non-empty")
|
||||
pipeline = cast(SupportsPromptUpdate, self.pipeline)
|
||||
pipeline.prepare_prompt_update(state, prompt, event_id, transition_chunks)
|
||||
@@ -50,6 +50,7 @@ from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput, KVPref
|
||||
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
from vllm_omni.diffusion.worker.utils import BaseRunnerOutput, BatchRunnerOutput
|
||||
from vllm_omni.engine.stage_init_utils import set_death_signal
|
||||
from vllm_omni.inputs.data import OmniInteractionPrompt
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
from vllm_omni.profiler import OmniTorchProfilerWrapper, create_omni_profiler
|
||||
@@ -511,6 +512,15 @@ class DiffusionWorker:
|
||||
# for the frontend server yet.
|
||||
return self.lora_manager.add_adapter(lora_request)
|
||||
|
||||
def submit_interaction(
|
||||
self,
|
||||
request_id: str,
|
||||
interaction: OmniInteractionPrompt,
|
||||
) -> None:
|
||||
"""Apply a midway interaction to an active stepwise request."""
|
||||
assert self.model_runner is not None, "Model runner not initialized"
|
||||
self.model_runner.submit_interaction(request_id, interaction)
|
||||
|
||||
def list_loras(self) -> list[int]:
|
||||
return self.lora_manager.list_adapters()
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ from dataclasses import dataclass, field
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.prompt_update import prompt_update_versions
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
|
||||
@@ -535,12 +536,17 @@ def _same_composition(
|
||||
cached_batch: InputBatch | None,
|
||||
request_ids: list[str],
|
||||
idx_mapping_np: np.ndarray,
|
||||
states: Sequence[StepRequestState],
|
||||
) -> bool:
|
||||
if cached_batch is None:
|
||||
return False
|
||||
if cached_batch.request_ids != request_ids:
|
||||
return False
|
||||
return np.array_equal(cached_batch.idx_mapping_np, idx_mapping_np)
|
||||
if not np.array_equal(cached_batch.idx_mapping_np, idx_mapping_np):
|
||||
return False
|
||||
# Midway prompt updates (typically for video generation) can change prompt_embeds without changing ids/mapping.
|
||||
# In this case, each request state manages the embedding's "version". Use it to determine if cache is still valid.
|
||||
return cached_batch._prompt_update_versions == prompt_update_versions(states)
|
||||
|
||||
|
||||
def _scatter_batch_tensor_by_mapping(
|
||||
@@ -615,6 +621,10 @@ class InputBatch:
|
||||
negative_txt_seq_lens: list[int] | None = None
|
||||
states: Sequence[StepRequestState] = field(default_factory=tuple)
|
||||
|
||||
# For midway prompt updates (typically for video generation) that changes embeddings without changing ids,
|
||||
# Keep a snapshot of the current per-request versions of prompt embeddings for later runtime comparison
|
||||
_prompt_update_versions: tuple[int, ...] = ()
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if len(self.request_ids) != int(self.idx_mapping.numel()):
|
||||
raise ValueError("`request_ids` and `idx_mapping` must have the same length.")
|
||||
@@ -653,6 +663,7 @@ class InputBatch:
|
||||
self.img_shapes = _prepare_img_shapes(states)
|
||||
self.txt_seq_lens = _prepare_seq_lens(states, "txt_seq_lens")
|
||||
self.negative_txt_seq_lens = _prepare_seq_lens(states, "negative_txt_seq_lens")
|
||||
self._prompt_update_versions = prompt_update_versions(states)
|
||||
|
||||
def _repack_dynamic_fields(
|
||||
self,
|
||||
@@ -693,7 +704,7 @@ class InputBatch:
|
||||
selected_states, idx_mapping, idx_mapping_np = _select_states(states, idx_mapping)
|
||||
request_ids = _prepare_request_ids(selected_states)
|
||||
|
||||
if _same_composition(cached_batch, request_ids, idx_mapping_np):
|
||||
if _same_composition(cached_batch, request_ids, idx_mapping_np, selected_states):
|
||||
assert cached_batch is not None
|
||||
cached_batch._repack_dynamic_fields(selected_states)
|
||||
return cached_batch
|
||||
@@ -733,6 +744,7 @@ class InputBatch:
|
||||
"negative_txt_seq_lens",
|
||||
),
|
||||
states=tuple(selected_states),
|
||||
_prompt_update_versions=prompt_update_versions(selected_states),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -48,6 +48,9 @@ def split_diffusion_output_by_request(
|
||||
peak_memory_mb=result.peak_memory_mb,
|
||||
chunk_index=result.chunk_index,
|
||||
total_chunks=result.total_chunks,
|
||||
started_event_ids=result.started_event_ids,
|
||||
active_event_ids=result.active_event_ids,
|
||||
completed_event_ids=result.completed_event_ids,
|
||||
)
|
||||
for idx in range(req.num_reqs)
|
||||
]
|
||||
|
||||
Reference in new issue
Block a user