[Frontend] Add midway prompt update for streaming video generation (#4652)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
Zeyu Huang | 黃澤宇 authored and GitHub committed 2026-07-27 14:07:43 +08:00
1 parent 1f01d05603
commit eaf90d8eff
28 files changed
+2615 -133

No files matched your search

@@ -14,7 +14,7 @@ import copy
import time
from collections.abc import Callable
from contextlib import AbstractContextManager, nullcontext
from typing import Any
from typing import TYPE_CHECKING, Any, cast
import torch
from torch.profiler import record_function
@@ -32,7 +32,7 @@ from vllm_omni.diffusion.compile import regionally_compile
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.forward_context import set_forward_context
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import supports_step_execution
from vllm_omni.diffusion.models.interface import SupportsPromptUpdate, supports_prompt_update, supports_step_execution
from vllm_omni.diffusion.offloader import get_offload_backend
from vllm_omni.diffusion.registry import _NO_CACHE_ACCELERATION
from vllm_omni.diffusion.request import OmniDiffusionRequest
@@ -53,6 +53,9 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
from vllm_omni.worker.omni_connector_model_runner_mixin import OmniConnectorModelRunnerMixin
if TYPE_CHECKING:
from vllm_omni.inputs.data import OmniInteractionPrompt
logger = init_logger(__name__)
@@ -803,3 +806,48 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
self._update_states_after(states, input_batch, pipeline_interrupted)
return BatchRunnerOutput.from_list(runner_output_list)
def submit_interaction(
self,
request_id: str,
interaction: OmniInteractionPrompt,
) -> None:
"""Route a midway interaction to the matching active stepwise request feature."""
assert self.pipeline is not None, "Model not loaded. Call load_model() first."
if not self.od_config.streaming_output:
raise ValueError("submit_interaction requires streaming_output=True")
if not self._supports_step_mode():
raise ValueError("submit_interaction requires step execution support")
event = interaction.get("event")
if isinstance(event, dict) and "prompt" in event and "multi_modal_data" not in event:
# Is a prompt update interaction.
self._submit_prompt_update_interaction(request_id, interaction)
return
raise NotImplementedError(
"Only text-only prompt update interactions with 'event.prompt' and optional "
"'transition_chunks' are supported in this release"
)
def _submit_prompt_update_interaction(
self,
request_id: str,
interaction: OmniInteractionPrompt,
) -> None:
"""Queue a prompt-update interaction for an active stepwise request."""
if not supports_prompt_update(self.pipeline):
raise ValueError(f"prompt_update is not supported by pipeline {self.od_config.model_class_name!r}")
state = self.state_cache.get(request_id)
if state is None:
raise ValueError(f"No active request state for prompt_update: {request_id!r}")
event = cast(dict[str, Any], interaction.get("event"))
prompt = event["prompt"]
transition_chunks = interaction.get("transition_chunks")
event_id = interaction.get("event_id")
if not isinstance(event_id, str) or not event_id:
raise ValueError("event_id must be non-empty")
pipeline = cast(SupportsPromptUpdate, self.pipeline)
pipeline.prepare_prompt_update(state, prompt, event_id, transition_chunks)
@@ -50,6 +50,7 @@ from vllm_omni.diffusion.sched.interface import DiffusionSchedulerOutput, KVPref
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
from vllm_omni.diffusion.worker.utils import BaseRunnerOutput, BatchRunnerOutput
from vllm_omni.engine.stage_init_utils import set_death_signal
from vllm_omni.inputs.data import OmniInteractionPrompt
from vllm_omni.lora.request import LoRARequest
from vllm_omni.platforms import current_omni_platform
from vllm_omni.profiler import OmniTorchProfilerWrapper, create_omni_profiler
@@ -511,6 +512,15 @@ class DiffusionWorker:
# for the frontend server yet.
return self.lora_manager.add_adapter(lora_request)
def submit_interaction(
self,
request_id: str,
interaction: OmniInteractionPrompt,
) -> None:
"""Apply a midway interaction to an active stepwise request."""
assert self.model_runner is not None, "Model runner not initialized"
self.model_runner.submit_interaction(request_id, interaction)
def list_loras(self) -> list[int]:
return self.lora_manager.list_adapters()
+14 -2
View File
@@ -17,6 +17,7 @@ from dataclasses import dataclass, field
import numpy as np
import torch
from vllm_omni.diffusion.prompt_update import prompt_update_versions
from vllm_omni.diffusion.worker.utils import StepRequestState
@@ -535,12 +536,17 @@ def _same_composition(
cached_batch: InputBatch | None,
request_ids: list[str],
idx_mapping_np: np.ndarray,
states: Sequence[StepRequestState],
) -> bool:
if cached_batch is None:
return False
if cached_batch.request_ids != request_ids:
return False
return np.array_equal(cached_batch.idx_mapping_np, idx_mapping_np)
if not np.array_equal(cached_batch.idx_mapping_np, idx_mapping_np):
return False
# Midway prompt updates (typically for video generation) can change prompt_embeds without changing ids/mapping.
# In this case, each request state manages the embedding's "version". Use it to determine if cache is still valid.
return cached_batch._prompt_update_versions == prompt_update_versions(states)
def _scatter_batch_tensor_by_mapping(
@@ -615,6 +621,10 @@ class InputBatch:
negative_txt_seq_lens: list[int] | None = None
states: Sequence[StepRequestState] = field(default_factory=tuple)
# For midway prompt updates (typically for video generation) that changes embeddings without changing ids,
# Keep a snapshot of the current per-request versions of prompt embeddings for later runtime comparison
_prompt_update_versions: tuple[int, ...] = ()
def __post_init__(self) -> None:
if len(self.request_ids) != int(self.idx_mapping.numel()):
raise ValueError("`request_ids` and `idx_mapping` must have the same length.")
@@ -653,6 +663,7 @@ class InputBatch:
self.img_shapes = _prepare_img_shapes(states)
self.txt_seq_lens = _prepare_seq_lens(states, "txt_seq_lens")
self.negative_txt_seq_lens = _prepare_seq_lens(states, "negative_txt_seq_lens")
self._prompt_update_versions = prompt_update_versions(states)
def _repack_dynamic_fields(
self,
@@ -693,7 +704,7 @@ class InputBatch:
selected_states, idx_mapping, idx_mapping_np = _select_states(states, idx_mapping)
request_ids = _prepare_request_ids(selected_states)
if _same_composition(cached_batch, request_ids, idx_mapping_np):
if _same_composition(cached_batch, request_ids, idx_mapping_np, selected_states):
assert cached_batch is not None
cached_batch._repack_dynamic_fields(selected_states)
return cached_batch
@@ -733,6 +744,7 @@ class InputBatch:
"negative_txt_seq_lens",
),
states=tuple(selected_states),
_prompt_update_versions=prompt_update_versions(selected_states),
)
@@ -48,6 +48,9 @@ def split_diffusion_output_by_request(
peak_memory_mb=result.peak_memory_mb,
chunk_index=result.chunk_index,
total_chunks=result.total_chunks,
started_event_ids=result.started_event_ids,
active_event_ids=result.active_event_ids,
completed_event_ids=result.completed_event_ids,
)
for idx in range(req.num_reqs)
]