[Core][Diffusion] Camera interaction for diffusion streaming generation (LingBot World 2 as example) (#7198)
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
This commit is contained in:
1 parent
139a47a578
commit
e36babd48f
26 files changed
+2071
-200
No files matched your search
@@ -535,19 +535,23 @@ class HeliosPipeline(
|
||||
"zero_steps": int(extra.get("zero_steps", 1)),
|
||||
}
|
||||
)
|
||||
self.prepare_next_chunk(state)
|
||||
return state
|
||||
|
||||
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
|
||||
"""Expose this chunk's decoded media extent for interaction timelines."""
|
||||
num_frames = int(state.extra["window_num_frames"])
|
||||
num_media_frames = int(state.extra["window_num_frames"])
|
||||
num_latent_frames = int(state.extra["num_latent_frames_per_chunk"])
|
||||
fps = state.sampling.fps
|
||||
if fps is None or float(fps) <= 0:
|
||||
raise ValueError(
|
||||
"sampling.fps is required and must be > 0 for interaction modalities that use the "
|
||||
f"chunk media timeline, got {fps!r} (request_id={state.request_id!r})"
|
||||
)
|
||||
return ChunkMediaSpec(num_frames=num_frames, fps=float(fps))
|
||||
return ChunkMediaSpec(
|
||||
num_media_frames=num_media_frames,
|
||||
fps=float(fps),
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
|
||||
@override
|
||||
def prepare_next_chunk(self, state: StepRequestState) -> None:
|
||||
|
||||
@@ -151,10 +151,10 @@ class SupportsInteractionApply(Protocol):
|
||||
"""Optional protocol for pipelines with unified mid-generation, chunk-boundary hooks."""
|
||||
|
||||
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
|
||||
"""Return the media timeline represented by the upcoming/current chunk.
|
||||
"""Return the media timeline and latent frame count for the upcoming/current chunk.
|
||||
|
||||
Useful when interaction handler needs interpolation/integration on a frame-by-frame basis,
|
||||
or for backpressure/pacing.
|
||||
and/or for backpressure/pacing.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm_omni.experimental.ar_diffusion.tick_protocol import (
|
||||
|
||||
LINGBOT_CAMERA_ACTION_SCHEMA = "lingbot.camera_actions.v1"
|
||||
LINGBOT_CAMERA_TRAJECTORY_SCHEMA = "lingbot.camera_trajectory.v1"
|
||||
LINGBOT_CONTROLLER_TRANSLATION_UNIT = 0.05
|
||||
_ACTION_ORDER = ("w", "a", "s", "d", "i", "j", "k", "l")
|
||||
_VALID_ACTIONS = frozenset(_ACTION_ORDER)
|
||||
_REFERENCE_HEIGHT = 480
|
||||
@@ -282,7 +283,7 @@ def integrate_lingbot_camera_actions(
|
||||
"""Convert latent-frame WASD/IJKL controls to cumulative C2W poses.
|
||||
|
||||
Calibration and motion constants intentionally match SGLang's LingBot
|
||||
adapter: movement 0.05, pitch 4 degrees, yaw 6 degrees, pitch clamp 85
|
||||
adapter: movement 0.05 (LINGBOT_CONTROLLER_TRANSLATION_UNIT), pitch 4 degrees, yaw 6 degrees, pitch clamp 85
|
||||
degrees, and target-resolution focal lengths of 500 pixels.
|
||||
"""
|
||||
|
||||
@@ -325,13 +326,31 @@ def integrate_lingbot_camera_actions(
|
||||
right /= right_norm + 1e-6
|
||||
|
||||
movement = np.zeros(3, dtype=np.float64)
|
||||
movement += forward * 0.05 * (("w" in actions) - ("s" in actions))
|
||||
movement += right * 0.05 * (("d" in actions) - ("a" in actions))
|
||||
movement += forward * LINGBOT_CONTROLLER_TRANSLATION_UNIT * (("w" in actions) - ("s" in actions))
|
||||
movement += right * LINGBOT_CONTROLLER_TRANSLATION_UNIT * (("d" in actions) - ("a" in actions))
|
||||
# Keep the cumulative controller pose in FP64 between chunks, matching
|
||||
# SGLang's NumPy integration and avoiding long-session drift.
|
||||
current_pose = np.eye(4, dtype=np.float64)
|
||||
current_pose[:3, :3] = new_rotation
|
||||
current_pose[:3, 3] = translation + movement
|
||||
poses.append(current_pose)
|
||||
|
||||
trajectory = camera_trajectory_from_absolute_pose(torch.from_numpy(np.stack(poses)), width=width, height=height)
|
||||
return trajectory, current_pitch
|
||||
|
||||
|
||||
def camera_trajectory_from_absolute_pose(
|
||||
poses: torch.Tensor,
|
||||
*,
|
||||
width: int,
|
||||
height: int,
|
||||
) -> CameraTrajectory:
|
||||
if width <= 0 or height <= 0:
|
||||
raise ValueError("camera action resolution must be positive.")
|
||||
if poses.ndim != 3 or poses.shape[-2:] != (4, 4):
|
||||
raise ValueError(f"absolute camera poses must have shape [frames, 4, 4], got {tuple(poses.shape)}.")
|
||||
if poses.shape[0] == 0:
|
||||
raise ValueError("absolute camera poses must not be empty.")
|
||||
# build_plucker_embedding expects intrinsics in the 832x480 reference
|
||||
# coordinate system. These values become SGLang's [500, 500, W/2, H/2]
|
||||
# after that function scales them to the requested resolution.
|
||||
@@ -342,9 +361,7 @@ def integrate_lingbot_camera_actions(
|
||||
_REFERENCE_HEIGHT / 2,
|
||||
)
|
||||
trajectory = CameraTrajectory(
|
||||
# Keep the cumulative controller pose in FP64 between chunks, matching
|
||||
# SGLang's NumPy integration and avoiding long-session drift.
|
||||
poses=torch.from_numpy(np.stack(poses)),
|
||||
poses=poses,
|
||||
intrinsics=torch.tensor(reference_intrinsics, dtype=torch.float32).repeat(len(poses), 1),
|
||||
)
|
||||
return trajectory, current_pitch
|
||||
return trajectory
|
||||
@@ -389,7 +389,11 @@ def _invert_camera_poses(poses: torch.Tensor) -> torch.Tensor:
|
||||
return inverse
|
||||
|
||||
|
||||
def _prepare_framewise_poses(raw_poses: torch.Tensor) -> torch.Tensor:
|
||||
def _prepare_framewise_poses(
|
||||
raw_poses: torch.Tensor,
|
||||
*,
|
||||
translation_scale: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Convert raw C2W poses to normalized first-relative framewise deltas."""
|
||||
|
||||
# Remove global placement, then convert the sequence to framewise deltas.
|
||||
@@ -402,12 +406,19 @@ def _prepare_framewise_poses(raw_poses: torch.Tensor) -> torch.Tensor:
|
||||
relative_poses[1:],
|
||||
)
|
||||
|
||||
# Only relative direction/magnitude matters to the checkpoint; normalizing
|
||||
# by the largest step removes the trajectory's arbitrary translation scale.
|
||||
# Normalizing the control's scale. Different between scripted and realtime input.
|
||||
translations = relative_poses[:, :3, 3]
|
||||
max_translation_norm = torch.linalg.vector_norm(translations, dim=-1).max()
|
||||
if max_translation_norm > 0:
|
||||
relative_poses[:, :3, 3] = translations / max_translation_norm
|
||||
if translation_scale is None:
|
||||
# Scripted action_path input: divide by the largest step so an arbitrary recording scale becomes 1.
|
||||
scale = torch.linalg.vector_norm(translations, dim=-1).max()
|
||||
else:
|
||||
# Realtime interaction input: use the controller translation unit (one WASD step = full speed).
|
||||
# It cannot see future motion or normalize globally.
|
||||
if translation_scale <= 0:
|
||||
raise ValueError(f"translation_scale must be > 0, got {translation_scale}")
|
||||
scale = translation_scale
|
||||
if float(scale) > 0:
|
||||
relative_poses[:, :3, 3] = translations / scale
|
||||
return relative_poses
|
||||
|
||||
|
||||
@@ -420,6 +431,7 @@ def build_plucker_embedding(
|
||||
target_width: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
translation_scale: float | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Build checkpoint-ordered ``(ray origin, ray direction)`` channels."""
|
||||
|
||||
@@ -438,7 +450,10 @@ def build_plucker_embedding(
|
||||
|
||||
# Compute geometry in at least FP32, then cast at the model boundary.
|
||||
compute_dtype = torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype
|
||||
poses = _prepare_framewise_poses(trajectory.poses.to(device=device, dtype=compute_dtype))
|
||||
poses = _prepare_framewise_poses(
|
||||
trajectory.poses.to(device=device, dtype=compute_dtype),
|
||||
translation_scale=translation_scale,
|
||||
)
|
||||
intrinsics = trajectory.intrinsics.to(device=device, dtype=compute_dtype).clone()
|
||||
intrinsics[:, (0, 2)] *= width / _REFERENCE_WIDTH
|
||||
intrinsics[:, (1, 3)] *= height / _REFERENCE_HEIGHT
|
||||
|
||||
@@ -26,16 +26,21 @@ from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import DistributedAutoencoderKLWan
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.interaction.mixin import InteractionMixin
|
||||
from vllm_omni.diffusion.interaction.modality_handlers.camera import CameraSession
|
||||
from vllm_omni.diffusion.interaction.types import ChunkMediaSpec
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders
|
||||
from vllm_omni.diffusion.models.interface import SupportImageInput, SupportsComponentDiscovery, SupportsStepExecution
|
||||
from vllm_omni.diffusion.models.lingbot_world.actions import (
|
||||
LINGBOT_CAMERA_ACTION_SCHEMA,
|
||||
LINGBOT_CAMERA_TRAJECTORY_SCHEMA,
|
||||
LINGBOT_CONTROLLER_TRANSLATION_UNIT,
|
||||
LingBotCameraActionFrames,
|
||||
LingBotCameraActionScript,
|
||||
as_camera_action_frames,
|
||||
as_camera_action_script,
|
||||
camera_trajectory_from_absolute_pose,
|
||||
integrate_lingbot_camera_actions,
|
||||
parse_lingbot_camera_action_frames,
|
||||
parse_lingbot_camera_action_script,
|
||||
@@ -385,25 +390,36 @@ def get_lingbot_world_pre_process_func(
|
||||
camera_actions = None
|
||||
else:
|
||||
action_path = extra_args.get("action_path")
|
||||
if not isinstance(action_path, (str, os.PathLike)) or not str(action_path):
|
||||
raise ValueError("action_path is required in sampling_params.extra_args.action_path.")
|
||||
if not configured_action_root:
|
||||
raise ValueError(
|
||||
"sampling_params.extra_args.action_path requires a trusted action root configured by "
|
||||
f"model_config.lingbot_action_root or {_ACTION_ROOT_ENV}."
|
||||
if action_path is None or action_path == "":
|
||||
# Stepwise / streaming sessions may omit a request-scoped camera
|
||||
# script and instead drive motion via mid-generation camera
|
||||
# interaction (idle hold until the first camera event).
|
||||
if getattr(od_config, "step_execution", False) or getattr(od_config, "streaming_output", False):
|
||||
trajectory = None
|
||||
camera_actions = None
|
||||
camera_action_script = None
|
||||
else:
|
||||
raise ValueError("action_path is required in sampling_params.extra_args.action_path.")
|
||||
else:
|
||||
if not isinstance(action_path, (str, os.PathLike)) or not str(action_path):
|
||||
raise ValueError("action_path is required in sampling_params.extra_args.action_path.")
|
||||
if not configured_action_root:
|
||||
raise ValueError(
|
||||
"sampling_params.extra_args.action_path requires a trusted action root configured by "
|
||||
f"model_config.lingbot_action_root or {_ACTION_ROOT_ENV}."
|
||||
)
|
||||
action_directory = resolve_trusted_action_directory(
|
||||
action_path,
|
||||
configured_action_root,
|
||||
)
|
||||
action_directory = resolve_trusted_action_directory(
|
||||
action_path,
|
||||
configured_action_root,
|
||||
)
|
||||
try:
|
||||
trajectory = load_camera_trajectory(action_directory)
|
||||
except OSError:
|
||||
raise ValueError(
|
||||
"Unable to load camera trajectory from action_path; expected poses.npy and intrinsics.npy."
|
||||
) from None
|
||||
camera_actions = None
|
||||
camera_action_script = None
|
||||
try:
|
||||
trajectory = load_camera_trajectory(action_directory)
|
||||
except OSError:
|
||||
raise ValueError(
|
||||
"Unable to load camera trajectory from action_path; expected poses.npy and intrinsics.npy."
|
||||
) from None
|
||||
camera_actions = None
|
||||
camera_action_script = None
|
||||
|
||||
updated_prompt = dict(prompt)
|
||||
updated_multi_modal_data = dict(multi_modal_data)
|
||||
@@ -460,6 +476,7 @@ class LingBotWorldCausalDMDPipeline(
|
||||
SupportImageInput,
|
||||
SupportsComponentDiscovery,
|
||||
SupportsStepExecution,
|
||||
InteractionMixin,
|
||||
ProgressBarMixin,
|
||||
DiffusionPipelineProfilerMixin,
|
||||
):
|
||||
@@ -779,8 +796,10 @@ class LingBotWorldCausalDMDPipeline(
|
||||
camera_sources = (
|
||||
int(camera_trajectory is not None) + int(camera_actions is not None) + int(camera_action_script is not None)
|
||||
)
|
||||
if camera_sources != 1:
|
||||
raise ValueError("LingBot pre-processing must materialize exactly one camera input.")
|
||||
if camera_sources > 1:
|
||||
raise ValueError("LingBot pre-processing must materialize at most one camera input.")
|
||||
# Zero camera sources are allowed for stepwise camera-interaction sessions;
|
||||
# request-mode ``forward`` still requires an explicit trajectory or actions.
|
||||
|
||||
request_flow_shift = (
|
||||
extra_args["flow_shift"] if "flow_shift" in extra_args else getattr(self.scheduler.config, "shift", 5.0)
|
||||
@@ -1047,7 +1066,7 @@ class LingBotWorldCausalDMDPipeline(
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
previous: CameraTrajectory | None = None,
|
||||
latent_aligned: bool = False,
|
||||
latent_aligned: bool = False, # iff using realtime interaction inputs
|
||||
) -> tuple[torch.Tensor, CameraTrajectory]:
|
||||
"""Convert raw camera frames to a latent-aligned ray tensor."""
|
||||
|
||||
@@ -1083,10 +1102,10 @@ class LingBotWorldCausalDMDPipeline(
|
||||
),
|
||||
)
|
||||
drop_anchor = True
|
||||
elif inputs.camera_actions is not None:
|
||||
# The action integrator returns post-action poses. Keep the first
|
||||
# action visible to framewise-delta conditioning by prepending its
|
||||
# known pre-action state (identity for a new realtime session).
|
||||
elif inputs.camera_actions is not None or latent_aligned:
|
||||
# Action/interaction integrators return post-action poses. Prepend the
|
||||
# known pre-action state (identity for a new session) so the first
|
||||
# pose still contributes a framewise delta.
|
||||
identity = torch.eye(
|
||||
4,
|
||||
device=trajectory.poses.device,
|
||||
@@ -1100,6 +1119,8 @@ class LingBotWorldCausalDMDPipeline(
|
||||
),
|
||||
)
|
||||
drop_anchor = True
|
||||
# Realtime interaction cannot see future steps; use the controller translation unit to normalize speed.
|
||||
# Full action_path keeps max-norm.
|
||||
camera_embedding = build_plucker_embedding(
|
||||
embedding_trajectory,
|
||||
height=inputs.height,
|
||||
@@ -1108,6 +1129,7 @@ class LingBotWorldCausalDMDPipeline(
|
||||
target_width=inputs.width,
|
||||
device=self.device,
|
||||
dtype=dtype,
|
||||
translation_scale=LINGBOT_CONTROLLER_TRANSLATION_UNIT if latent_aligned else None,
|
||||
)
|
||||
if drop_anchor:
|
||||
camera_embedding = camera_embedding[1:]
|
||||
@@ -1277,6 +1299,11 @@ class LingBotWorldCausalDMDPipeline(
|
||||
"request mode takes one three-frame camera_actions control per "
|
||||
"block, or a camera_trajectory for the request."
|
||||
)
|
||||
if inputs.camera_trajectory is None and inputs.camera_actions is None:
|
||||
raise ValueError(
|
||||
"LingBot request mode requires camera_trajectory or camera_actions; "
|
||||
"omit them only for stepwise camera-interaction sessions."
|
||||
)
|
||||
tick = ARDiffusionTickRequest.from_extra_args(req.sampling_params.extra_args)
|
||||
if tick is not None and self._ar_diffusion_kv_state is None:
|
||||
raise RuntimeError("LingBot typed ticks require ARDiffusionEngine session binding.")
|
||||
@@ -1645,11 +1672,6 @@ class LingBotWorldCausalDMDPipeline(
|
||||
}
|
||||
self._ar_text_caches(prompt_embeds, invalidate=False)
|
||||
self._ar_sessions[state.request_id] = _LingBotARSessionState()
|
||||
try:
|
||||
self._prepare_next_chunk(state)
|
||||
except Exception:
|
||||
self._release_condition_encoder_state(state.request_id)
|
||||
raise
|
||||
return state
|
||||
|
||||
def _prepare_next_chunk(self, state: StepRequestState) -> None:
|
||||
@@ -1668,7 +1690,14 @@ class LingBotWorldCausalDMDPipeline(
|
||||
dtype=extra["dtype"],
|
||||
)
|
||||
previous = extra.get("camera_tail")
|
||||
|
||||
if extra.get("camera_action_script") is not None:
|
||||
if (
|
||||
(camera_session := state.interaction_sessions.get("camera"))
|
||||
and isinstance(camera_session, CameraSession)
|
||||
and camera_session.has_received_input
|
||||
):
|
||||
raise ValueError("Cannot use mid-generation camera interaction together with camera_action_script; ")
|
||||
chunk_actions = extra["camera_action_script"][state.chunk_index]
|
||||
action_trajectory, camera_pitch = integrate_lingbot_camera_actions(
|
||||
chunk_actions,
|
||||
@@ -1678,11 +1707,12 @@ class LingBotWorldCausalDMDPipeline(
|
||||
initial_pitch=float(extra.get("camera_pitch", 0.0)),
|
||||
)
|
||||
extra["camera_pitch"] = camera_pitch
|
||||
media_frames = self._chunk_media_frame_count(state, block_frames=block_frames)
|
||||
chunk_inputs = replace(
|
||||
inputs,
|
||||
camera_trajectory=action_trajectory,
|
||||
camera_actions=chunk_actions,
|
||||
num_frames=(block_frames - 1) * self.vae_scale_factor_temporal + 1,
|
||||
num_frames=media_frames,
|
||||
num_latent_frames=block_frames,
|
||||
)
|
||||
camera, camera_tail = self._prepare_camera(
|
||||
@@ -1691,14 +1721,53 @@ class LingBotWorldCausalDMDPipeline(
|
||||
previous=previous,
|
||||
)
|
||||
elif extra.get("camera_embedding_cache") is not None:
|
||||
# Same slice request mode takes from its one full-trajectory
|
||||
# embedding, so both paths condition a block identically.
|
||||
if (
|
||||
(camera_session := state.interaction_sessions.get("camera"))
|
||||
and isinstance(camera_session, CameraSession)
|
||||
and camera_session.has_received_input
|
||||
):
|
||||
raise ValueError("Cannot use mid-generation camera interaction together with camera_embedding_cache; ")
|
||||
trajectory = extra["camera_trajectory_cache"]
|
||||
camera = extra["camera_embedding_cache"][:, :, start_frame:stop_frame]
|
||||
camera_tail = CameraTrajectory(
|
||||
poses=trajectory.poses[stop_frame - 1 : stop_frame].clone(),
|
||||
intrinsics=trajectory.intrinsics[stop_frame - 1 : stop_frame].clone(),
|
||||
)
|
||||
elif self._interaction_coordinator is not None and self._interaction_coordinator.has_modality("camera"):
|
||||
# Poses come from the latest boundary apply: ``prepare_encode`` for
|
||||
# chunk 0, the diffusion runner for later chunks. This method only
|
||||
# digests them — it does not apply interactions itself.
|
||||
camera_session = state.interaction_sessions.get("camera")
|
||||
if not isinstance(camera_session, CameraSession) or camera_session.last_absolute_poses is None:
|
||||
raise RuntimeError(
|
||||
"LingBot camera interaction requires apply_interaction_at_chunk_boundary before prepare_next_chunk."
|
||||
)
|
||||
absolute_poses = camera_session.last_absolute_poses
|
||||
media_frames = self._chunk_media_frame_count(state, block_frames=block_frames)
|
||||
num_latent_frames = block_frames
|
||||
if int(absolute_poses.shape[0]) != num_latent_frames:
|
||||
raise ValueError(
|
||||
"camera interaction must produce exactly one pose per latent frame; "
|
||||
f"got {int(absolute_poses.shape[0])}, expected {num_latent_frames}."
|
||||
)
|
||||
action_trajectory = camera_trajectory_from_absolute_pose(
|
||||
absolute_poses,
|
||||
width=inputs.width,
|
||||
height=inputs.height,
|
||||
)
|
||||
chunk_inputs = replace(
|
||||
inputs,
|
||||
camera_trajectory=action_trajectory,
|
||||
camera_actions=None,
|
||||
num_frames=media_frames,
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
camera, camera_tail = self._prepare_camera(
|
||||
chunk_inputs,
|
||||
dtype=extra["dtype"],
|
||||
previous=previous,
|
||||
latent_aligned=True,
|
||||
)
|
||||
else:
|
||||
raise RuntimeError("LingBot step execution is missing a camera trajectory or action script.")
|
||||
if camera.shape[2:] != condition.shape[2:]:
|
||||
@@ -1842,19 +1911,48 @@ class LingBotWorldCausalDMDPipeline(
|
||||
session_state.pending_encoder_cache = None
|
||||
session_state.next_chunk_index += 1
|
||||
state.chunk_index += 1
|
||||
finished = state.request_denoise_completed
|
||||
if not finished:
|
||||
self._prepare_next_chunk(state)
|
||||
else:
|
||||
self._release_condition_encoder_state(state.request_id)
|
||||
return DiffusionOutput(
|
||||
output=output,
|
||||
stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None,
|
||||
chunk_index=completed_chunk_index,
|
||||
total_chunks=state.total_chunks,
|
||||
finished=finished,
|
||||
finished=state.request_denoise_completed,
|
||||
)
|
||||
|
||||
def _chunk_media_frame_count(self, state: StepRequestState, *, block_frames: int) -> int:
|
||||
"""Pixel frames this chunk will emit when decoded.
|
||||
|
||||
Independent (non-streaming) decode restarts causal expansion every block,
|
||||
so each chunk is ``(block_frames - 1) * temporal + 1`` frames.
|
||||
Streaming decode expands only the session's opening latent to one frame and every later latent to the full
|
||||
temporal factor, so chunk 0 matches the independent count and later chunks are ``block_frames * temporal``.
|
||||
"""
|
||||
decode_state = self._streaming_decode_states.get(state.request_id)
|
||||
if decode_state is not None and decode_state.started:
|
||||
return block_frames * self.vae_scale_factor_temporal
|
||||
return (block_frames - 1) * self.vae_scale_factor_temporal + 1
|
||||
|
||||
def peek_chunk_media(self, state: StepRequestState) -> ChunkMediaSpec:
|
||||
"""Expose this chunk's decoded media extent and latent step count."""
|
||||
num_latent_frames = int(state.extra.get("block_frames") or self.transformer.config.num_frames_per_block)
|
||||
num_media_frames = self._chunk_media_frame_count(state, block_frames=num_latent_frames)
|
||||
fps = state.sampling.fps
|
||||
if fps is None or float(fps) <= 0:
|
||||
# Media-frame camera controls are not wall-clock paced; a unit fps keeps
|
||||
# resolve_event_frame_offset well-defined when the client omits sampling.fps.
|
||||
fps = float(num_media_frames)
|
||||
return ChunkMediaSpec(
|
||||
num_media_frames=num_media_frames,
|
||||
fps=fps,
|
||||
num_latent_frames=num_latent_frames,
|
||||
)
|
||||
|
||||
def prepare_next_chunk(self, state: StepRequestState) -> None:
|
||||
"""Prepare the next AR block after chunk-boundary interaction apply."""
|
||||
if state.request_denoise_completed:
|
||||
return
|
||||
self._prepare_next_chunk(state)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return cast(set[str], loader.load_weights(weights))
|
||||
Reference in new issue
Block a user