diff --git a/.gitignore b/.gitignore index 378de441c..40a6916b5 100644 --- a/.gitignore +++ b/.gitignore @@ -270,3 +270,6 @@ vllm_omni/_version.py .worktrees/ # CI overlay yamls materialized from tests/utils.py:_CI_OVERLAYS at test time tests/.ci_generated/ + +# Local Lance precision-matching dump scripts (not part of the repo) +lance_compare/ diff --git a/examples/offline_inference/lance/README.md b/examples/offline_inference/lance/README.md new file mode 100644 index 000000000..d029abdea --- /dev/null +++ b/examples/offline_inference/lance/README.md @@ -0,0 +1,52 @@ +# Lance (ByteDance) — offline inference + +[Lance](https://huggingface.co/bytedance-research/Lance) is a 3B unified +autoregressive + diffusion multimodal model on a Qwen2.5-VL backbone. It is +**BAGEL-lineage** (ByteDance Mixture-of-Transformers): the released `Lance_3B` +checkpoint uses the same `*_moe_gen` MoT weight layout as BAGEL, so vLLM-Omni +implements it by reusing the BAGEL transformer core and specializing only the +ViT (Qwen2.5-VL vision), the VAE (Wan2.2) and the checkpoint layout. + +This example covers all six Lance modalities from the upstream HF model card: +`t2i`, `t2v`, `image_edit`, `video_edit`, `x2t_image` (image understanding) and +`x2t_video` (video understanding). + +## Hardware + +Single NVIDIA GPU with 16 GB+ VRAM in BF16 (we test on B300 / A100). CUDA ≥ 12.4. + +## Run + +```bash +# Text-to-image +python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance \ + --prompts "a corgi astronaut on the moon, cinematic" \ + --steps 30 --cfg-text-scale 4.0 --timestep-shift 3.5 \ + --height 1024 --width 1024 \ + --output ./out + +# Text-to-video (uses the Lance_3B_Video subfolder; see ``--modality`` +# choices for all six task variants) +python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance/Lance_3B_Video --modality text2video \ + --num-frames 25 --video-height 480 --video-width 768 \ + --prompts "a cat playing piano, cinematic" \ + --steps 30 --fps 8 --output ./out +``` + +`video_edit` requires `--model bytedance-research/Lance/Lance_3B_Video` so +the 3-D `latent_pos_embed` table is loaded; the other paths can point at +the top-level `bytedance-research/Lance` repo and resolve the right +sub-checkpoint automatically. + +The HF repo bundles everything (`Lance_3B/`, `Lance_3B_Video/`, +`Qwen2.5-VL-ViT/`, `Wan2.2_VAE.pth`); no separate downloads are required. + +## Defaults + +Matches upstream `inference_lance.sh`: 30 denoising steps, timestep-shift 3.5, +text CFG 4.0, seed 42, 1024×1024 (override with `--height` / `--width`). For +the understanding paths (`img2text` / `video2text`), sampling is enabled by +default at `--text-temperature 0.8` because Lance's greedy decoder emits an +immediate EOS for many prompts. diff --git a/examples/offline_inference/lance/end2end.py b/examples/offline_inference/lance/end2end.py new file mode 100644 index 000000000..504dceec6 --- /dev/null +++ b/examples/offline_inference/lance/end2end.py @@ -0,0 +1,339 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Lance-3B offline inference (single-stage). + +Lance is BAGEL-lineage; this example covers the single-stage paths wired in +this PR: + + text2img — image generation from text + img2text — image understanding (caption / VQA) + +Examples: + + python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance \ + --prompts "a corgi astronaut on the moon, cinematic" \ + --output ./out + + python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance --modality img2text \ + --image-path /path/to/photo.jpg \ + --prompts "Describe this image in detail." +""" + +import argparse +import os + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--model", default="bytedance-research/Lance", help="HF repo or local path.") + p.add_argument("--prompts", nargs="+", default=None, help="Input text prompts.") + p.add_argument("--txt-prompts", type=str, default=None, help="File with one prompt per line.") + p.add_argument( + "--modality", + default="text2img", + choices=["text2img", "img2text", "text2video", "video2text", "img2img", "video2video", "image2video"], + help="Lance single-stage modality.", + ) + p.add_argument("--image-path", type=str, default=None, help="Input image for img2text.") + p.add_argument("--video-path", type=str, default=None, help="Input video file for video2text.") + p.add_argument("--num-frames", type=int, default=25, help="Number of RGB frames for text2video (max 121).") + p.add_argument("--video-height", type=int, default=480, help="Video frame height.") + p.add_argument("--video-width", type=int, default=768, help="Video frame width.") + p.add_argument("--height", type=int, default=None, help="Image height (t2i). Default = max_hw (1024).") + p.add_argument("--width", type=int, default=None, help="Image width (t2i). Default = max_hw (1024).") + p.add_argument( + "--fps", type=int, default=12, help="Output video FPS when saving MP4 (matches upstream Lance's save_fps=12)." + ) + p.add_argument("--output", type=str, default=".", help="Output directory.") + p.add_argument("--steps", type=int, default=30, help="Denoising steps (Lance default 30).") + p.add_argument("--cfg-text-scale", type=float, default=4.0, help="Text CFG scale (Lance default 4.0).") + p.add_argument("--timestep-shift", type=float, default=3.5, help="Flow-match timestep shift (Lance default 3.5).") + p.add_argument("--negative-prompt", type=str, default=None) + p.add_argument("--seed", type=int, default=42) + p.add_argument("--max-text-tokens", type=int, default=512, help="Max tokens for img2text generation.") + p.add_argument( + "--do-sample", + action="store_true", + default=True, + help="Sample (vs greedy) for x2t generation; greedy frequently emits an immediate EOS for Lance.", + ) + p.add_argument("--no-sample", dest="do_sample", action="store_false", help="Disable sampling (greedy decoding).") + p.add_argument( + "--text-temperature", + type=float, + default=0.8, + help="Sampling temperature for x2t generation. Lance frequently emits an immediate EOS below ~0.7; 0.8 is a good default.", + ) + p.add_argument( + "--system-prompt", + type=str, + default=None, + help=( + "Override the task system prompt (used by x2t_image / x2t_video for " + 'per-example QA instructions like "Look at the image carefully and ' + 'answer the question."; without it x2t falls back to a ' + "caption-style default and the model describes instead of answering)." + ), + ) + # Lance is single-stage diffusion — no deploy YAML needed. Required + # engine knobs (``pipeline``, ``enforce_eager``, ``trust_remote_code``, + # ``max_num_seqs=1`` …) are passed as flat kwargs to ``Omni`` below + # and ``create_default_diffusion`` materializes the stage config. + + from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults + + nullify_stage_engine_defaults(p) + return p.parse_args() + + +from vllm_omni.diffusion.models.lance.prompts import ( # noqa: E402 + VIDEO_PAD, + VISION_END, + VISION_START, + render_lance_prompt, +) + +# Visual block matching upstream Lance: <|vision_start|><|video_pad|><|vision_end|> +# (Lance's renderer uses ``<|video_pad|>`` even for image inputs by default.) +_VISION_BLOCK = f"{VISION_START}{VIDEO_PAD}{VISION_END}" + + +def _format_text2img_prompts(prompts): + return [{"prompt": render_lance_prompt("t2i", p), "modalities": ["image"]} for p in prompts] + + +def _format_text2video_prompts(prompts, num_frames, video_h, video_w): + extra = {"num_frames": num_frames, "video_height": video_h, "video_width": video_w} + return [{"prompt": render_lance_prompt("t2v", p), "modalities": ["video"], "extra_args": extra} for p in prompts] + + +def _format_image2video_prompts(prompts, image_path, num_frames, video_h, video_w): + """Image-to-Video (no first-frame pin): image + text → long video. + + Passes the input image via ``multi_modal_data.first_frame``. The pipeline + treats it as a 1-frame reference (VAE+ViT prefill) and generates a fresh + multi-frame video at the requested ``num_frames`` × ``video_h`` × + ``video_w`` shape. + """ + import os as _os + + if not image_path or not _os.path.exists(image_path): + raise ValueError(f"image2video requires --image-path pointing to an existing file, got: {image_path}") + from PIL import Image as _Image + + img = _Image.open(image_path).convert("RGB") + extra = {"num_frames": num_frames, "video_height": video_h, "video_width": video_w} + return [ + { + "prompt": render_lance_prompt("i2v", p, vision_token=_VISION_BLOCK), + "multi_modal_data": {"first_frame": img}, + "modalities": ["video"], + "extra_args": extra, + } + for p in prompts + ] + + +def _format_img2img_prompts(prompts, image_path): + import os as _os + + if not image_path or not _os.path.exists(image_path): + raise ValueError(f"img2img requires --image-path pointing to an existing file, got: {image_path}") + from PIL import Image as _Image + + img = _Image.open(image_path).convert("RGB") + return [ + { + "prompt": render_lance_prompt("image_edit", p, vision_token=_VISION_BLOCK), + "multi_modal_data": {"img2img": img}, + "modalities": ["image"], + } + for p in prompts + ] + + +def _format_video2video_prompts(prompts, video_path, num_frames, video_h, video_w): + import os as _os + + if not video_path or not _os.path.exists(video_path): + raise ValueError(f"video2video requires --video-path pointing to an existing file, got: {video_path}") + + extra = {"num_frames": num_frames, "video_height": video_h, "video_width": video_w} + return [ + { + "prompt": render_lance_prompt("video_edit", p, vision_token=_VISION_BLOCK), + "multi_modal_data": {"video": video_path}, + "modalities": ["video"], + "extra_args": extra, + } + for p in prompts + ] + + +def _format_video2text_prompts(prompts, video_path, system_prompt=None): + import os as _os + + if not video_path or not _os.path.exists(video_path): + raise ValueError(f"video2text requires --video-path pointing to an existing file, got: {video_path}") + import imageio.v3 as iio + + video = iio.imread(video_path) + return [ + { + "prompt": render_lance_prompt("x2t_video", p, vision_token=_VISION_BLOCK, system_prompt=system_prompt), + "multi_modal_data": {"video": video}, + "modalities": ["text"], + } + for p in prompts + ] + + +def _format_img2text_prompts(prompts, image_path, system_prompt=None): + from PIL import Image + + if not image_path or not os.path.exists(image_path): + raise ValueError(f"img2text requires --image-path pointing to an existing file, got: {image_path}") + img = Image.open(image_path).convert("RGB") + return [ + { + "prompt": render_lance_prompt("x2t_image", p, vision_token=_VISION_BLOCK, system_prompt=system_prompt), + "multi_modal_data": {"image": img}, + "modalities": ["text"], + } + for p in prompts + ] + + +def main(): + args = parse_args() + os.makedirs(args.output, exist_ok=True) + + # Lance aligns more naturally with upstream when using FLASH_ATTN + # (upstream uses flash_attn_varlen_func; SDPA accumulates ~5x more + # numerical drift through the 36-layer Qwen2 stack on B300). Default + # to flash_attn when the user hasn't explicitly set a backend. + os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN") + + if args.txt_prompts: + with open(args.txt_prompts, encoding="utf-8") as f: + prompts = [ln.strip() for ln in f if ln.strip()] + elif args.modality == "img2text": + prompts = args.prompts or ["Describe this image in detail."] + else: + prompts = args.prompts or ["A cute cat, studio lighting, 4k"] + + from vllm_omni.entrypoints.omni import Omni + + # Video modalities require the Lance_3B_Video checkpoint (3D + # latent_pos_embed shaped (126976, 2048)). Without it, vllm-omni silently + # loads Lance_3B (4096, 2048) — the image-only table — and any t_lat >= 1 + # immediately indexes out of bounds. Rewrite the model path so + # LancePipeline._select_video_variant picks the video subfolder. + resolved_model = args.model + if args.modality in ("text2video", "video2video", "video2text", "image2video"): + if not str(resolved_model).rstrip("/").endswith("Lance_3B_Video"): + candidate = os.path.join(resolved_model, "Lance_3B_Video") + if os.path.isdir(candidate): + resolved_model = candidate + + omni_kwargs = vars(args).copy() + omni_kwargs["model"] = resolved_model + # Lance single-stage defaults (formerly in vllm_omni/deploy/lance.yaml): + omni_kwargs.setdefault("pipeline", "lance") + omni_kwargs.setdefault("max_num_batched_tokens", 32768) + omni_kwargs.setdefault("max_num_seqs", 1) + omni_kwargs.setdefault("enforce_eager", True) + omni_kwargs.setdefault("trust_remote_code", True) + omni_kwargs.setdefault("enable_prefix_caching", False) + omni_kwargs.setdefault("async_chunk", False) + omni = Omni(**omni_kwargs) + + if args.modality == "img2text": + formatted = _format_img2text_prompts(prompts, args.image_path, system_prompt=args.system_prompt) + elif args.modality == "text2video": + formatted = _format_text2video_prompts(prompts, args.num_frames, args.video_height, args.video_width) + elif args.modality == "video2text": + formatted = _format_video2text_prompts(prompts, args.video_path, system_prompt=args.system_prompt) + elif args.modality == "img2img": + formatted = _format_img2img_prompts(prompts, args.image_path) + elif args.modality == "video2video": + formatted = _format_video2video_prompts( + prompts, args.video_path, args.num_frames, args.video_height, args.video_width + ) + elif args.modality == "image2video": + formatted = _format_image2video_prompts( + prompts, args.image_path, args.num_frames, args.video_height, args.video_width + ) + else: + formatted = _format_text2img_prompts(prompts) + + params_list = omni.default_sampling_params_list + diffusion_params = params_list[0] # single-stage: one param set + diffusion_params.num_inference_steps = args.steps # type: ignore + if args.seed is not None: + diffusion_params.seed = args.seed # type: ignore + if args.height is not None: + diffusion_params.height = args.height # type: ignore + if args.width is not None: + diffusion_params.width = args.width # type: ignore + extra = getattr(diffusion_params, "extra_args", {}) or {} + extra["cfg_text_scale"] = args.cfg_text_scale + extra["timestep_shift"] = args.timestep_shift + if args.modality in ("img2text", "video2text"): + extra["max_think_tokens"] = args.max_text_tokens + extra["do_sample"] = args.do_sample + extra["text_temperature"] = args.text_temperature + if args.negative_prompt is not None: + extra["negative_prompt"] = args.negative_prompt + diffusion_params.extra_args = extra # type: ignore + + outputs = list(omni.generate(prompts=formatted, sampling_params_list=params_list)) + + if args.modality in ("img2text", "video2text"): + for i, req_output in enumerate(outputs): + text = getattr(req_output, "output", None) or getattr(req_output, "text", None) + if text is None and hasattr(req_output, "custom_output"): + text = (req_output.custom_output or {}).get("text_output") + print(f"[Output {i}] {text}") + return + + if args.modality in ("text2video", "video2video", "image2video"): + for i, req_output in enumerate(outputs): + custom = getattr(req_output, "custom_output", None) or {} + frames = custom.get("video_frames") + if not frames: + print(f"[Output {i}] no video frames returned") + continue + mp4_path = os.path.join(args.output, f"lance_{i}.mp4") + try: + import imageio.v3 as iio + import numpy as np + + arr = np.stack([np.asarray(f) for f in frames], axis=0) + iio.imwrite(mp4_path, arr, fps=args.fps, codec="libx264", quality=8) + print(f"[Output {i}] Saved MP4 ({len(frames)} frames, {arr.shape[1]}x{arr.shape[2]}) to {mp4_path}") + except Exception as exc: + print(f"[Output {i}] MP4 encode failed ({exc!r}); writing per-frame ONGs instead") + frame_dir = os.path.join(args.output, f"lance_{i}_frames") + os.makedirs(frame_dir, exist_ok=True) + for j, frame in enumerate(frames): + frame.save(os.path.join(frame_dir, f"{j:04d}.png")) + print(f" → {frame_dir} ({len(frames)} frames)") + return + + idx = 0 + for req_output in outputs: + images = getattr(req_output, "images", None) + if not images: + continue + for j, img in enumerate(images): + path = os.path.join(args.output, f"lance_{idx}_{j}.png") + img.save(path) + print(f"[Output] Saved image to {path}") + idx += 1 + + +if __name__ == "__main__": + main() diff --git a/examples/offline_inference/lance/gradio_demo.py b/examples/offline_inference/lance/gradio_demo.py new file mode 100644 index 000000000..7dfeb46c9 --- /dev/null +++ b/examples/offline_inference/lance/gradio_demo.py @@ -0,0 +1,1296 @@ +"""Unified Gradio demo for Lance × vllm-omni — all 7 tasks. + +Style mirrors upstream Lance's lance_gradio.py: + * header with Lance + vllm-omni logos + shield badges + * radio task selector (Video Generation / Video Edit / Video Understanding / + Image-to-Video / Image Generation / Image Edit / Image Understanding) + * aspect-ratio + resolution dropdowns (not free H/W sliders) + * two-column layout (inputs left, output right) + * advanced-params accordion at the bottom + +Backend: routes to one of two Omni instances based on task: + * Lance_3B (t2i, image_edit, x2t_image) + * Lance_3B_Video (t2v, i2v, video_edit, x2t_video) + +Run: + CUDA_VISIBLE_DEVICES=6,7 \\ + /home/zjy/code/vllm-src/.venv/bin/python \\ + examples/offline_inference/lance/gradio_demo.py \\ + --model /path/to/Lance/snapshots/ +""" + +from __future__ import annotations + +import argparse +import base64 +import os +import tempfile +from pathlib import Path + +import gradio as gr +import imageio.v3 as iio +import numpy as np +from PIL import Image + +# --------------------------------------------------------- constants ----- + +# Mirror upstream Lance's resolution/aspect tables. +# +# Per ``data/datasets_custom/validation_dataset.py``, upstream restricts +# aspect-ratio buckets to **6 fixed ratios** and computes the per-bucket +# canonical (W, H) via: +# +# max_area = resolution_vae ** 2 +# for each ar = w/h: +# w1 = round(sqrt(max_area * ar) / 16) * 16 +# h1 = round(w1 / ar / 16) * 16 +# (… pick the candidate whose actual ratio is closer to ``ar``) +# +# with ``resolution_vae = 480`` for ``video_360p``, ``640`` for +# ``video_480p``, ``768`` for ``image_768res``. The numbers below are +# computed exactly that way so this gradio demo lands on the same canvas +# upstream would for the same input image — no aspect-ratio drift. +ASPECT_RATIO_CHOICES = ["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"] +DEFAULT_VIDEO_AR = "16:9" +DEFAULT_IMAGE_AR = "1:1" + +VIDEO_360P_AR_TO_SIZE = { + "21:9": (752, 320), + "16:9": (624, 352), + "4:3": (560, 416), + "1:1": (480, 480), + "3:4": (416, 560), + "9:16": (352, 624), +} +VIDEO_480P_AR_TO_SIZE = { + "21:9": (976, 416), + "16:9": (848, 480), + "4:3": (752, 560), + "1:1": (640, 640), + "3:4": (560, 752), + "9:16": (480, 848), +} +IMAGE_AR_TO_SIZE = { + "21:9": (1152, 496), + "16:9": (1024, 576), + "4:3": (896, 672), + "1:1": (768, 768), + "3:4": (672, 896), + "9:16": (576, 1024), +} + +VIDEO_RES_CHOICES = [("360p", "video_360p"), ("480p", "video_480p")] +VIDEO_RES_MAP = {"video_360p": VIDEO_360P_AR_TO_SIZE, "video_480p": VIDEO_480P_AR_TO_SIZE} +DEFAULT_VIDEO_RES = "video_480p" +DEFAULT_TIMESTEPS = 30 # upstream: --validation_num_timesteps 30 +DEFAULT_TIMESTEP_SHIFT = 3.5 # upstream: --validation_timestep_shift 3.5 +DEFAULT_CFG = 4.0 # upstream: --cfg_text_scale 4.0 (NOT 8.0) +DEFAULT_SEED = 42 # upstream: --validation_data_seed 42 +DEFAULT_VIDEO_DURATION_SEC = 5 # 5s × 12 fps + 1 = 61 frames — snappy for interactive demo +MAX_VIDEO_DURATION_SEC = 10 +# Per-task duration defaults — quality knobs (cfg, steps, ts_shift) are +# locked to upstream values, but the **demo duration** is intentionally +# shorter than upstream's per-script ``num_frames`` to keep gradio +# responsive (upstream uses 121 frames for t2v/i2v = 10s ≈ 2 min/req on +# B300; demo defaults give the same per-frame fidelity in ~60 s). +# Users can extend up to 10s with the slider when they want +# byte-identical reproduction of upstream's official output. +DURATION_DEFAULT_PER_TASK: dict[str, int] = { + "t2v": 5, # ~61 frames; upstream uses 121 + "i2v": 5, # ~61 frames; upstream uses 121 + "video_edit": 4, # ~49 frames; matches upstream's 50 +} + +# Task radio choices (label, internal_key) +TASK_VIDEO_GEN = "Video Generation" +TASK_I2V_LABEL = "Image to Video" +TASK_VIDEO_EDIT = "Video Edit" +TASK_VIDEO_UND = "Video Understanding" +TASK_IMAGE_GEN = "Image Generation" +TASK_IMAGE_EDIT = "Image Edit" +TASK_IMAGE_UND = "Image Understanding" + +LABEL_TO_TASK = { + TASK_VIDEO_GEN: "t2v", + TASK_I2V_LABEL: "i2v", + TASK_VIDEO_EDIT: "video_edit", + TASK_VIDEO_UND: "x2t_video", + TASK_IMAGE_GEN: "t2i", + TASK_IMAGE_EDIT: "image_edit", + TASK_IMAGE_UND: "x2t_image", +} + +LANCE_HOMEPAGE_URL = "https://lance-project.github.io/" +LANCE_PAPER_URL = "http://arxiv.org/abs/2605.18678" +LANCE_HF_URL = "https://huggingface.co/bytedance-research/Lance" +LANCE_GITHUB_URL = "https://github.com/bytedance/Lance" +VLLM_OMNI_GITHUB_URL = "https://github.com/vllm-project/vllm-omni" + + +# --------------------------------------------------- module handles ----- + +_OMNI_IMAGE = None +_OMNI_VIDEO = None +_PARAMS_IMAGE = None +_PARAMS_VIDEO = None + + +# ----------------------------------------------------------- args ----- + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--model", required=True, help="Parent snapshot dir containing Lance_3B and Lance_3B_Video subdirs.") + p.add_argument("--host", default="0.0.0.0") + p.add_argument("--port", type=int, default=7860) + p.add_argument("--share", action="store_true") + p.add_argument( + "--examples-root", + default="/home/zjy/code/lance-upstream", + help=( + "Path to a Lance upstream checkout containing ``config/examples/*.json`` " + "and the referenced media assets. When present, each task tab gets a " + "clickable list of official examples that auto-fill the prompt + media. " + "Set to '' to disable." + ), + ) + p.add_argument( + "--ulysses-degree", + type=int, + default=1, + help=( + "Ulysses sequence-parallel degree for *each* Omni instance " + "(image and video). Needs ``2 * ulysses_degree`` GPUs exposed via " + "CUDA_VISIBLE_DEVICES. NOTE: Lance currently lacks an ``_sp_plan`` " + "registry and the i2v first-frame pin is not SP-aware, so SP >1 is " + "experimental — t2i/t2v/edit/x2t work but i2v crashes. Prefer " + "``--replicas-per-omni`` for throughput scaling instead." + ), + ) + p.add_argument( + "--replicas-per-omni", + type=int, + default=1, + help=( + "Spawn ``N`` parallel replicas of each Omni (image + video) across " + "``2*N`` GPUs. Default ``1`` = the P0 layout (2 GPUs total, image " + "on GPU 0, video on GPU 1). Set ``4`` to use all 8 GPUs for " + "throughput; each replica loads a fresh model copy so startup time " + "and memory scale linearly." + ), + ) + return p.parse_args() + + +# -------------------------------------------------- model loading ----- + + +def _resolve_ckpts(snapshot: str): + img = Path(snapshot) / "Lance_3B" + vid = Path(snapshot) / "Lance_3B_Video" + if not img.is_dir(): + raise FileNotFoundError(f"Lance_3B not found under {snapshot}") + if not vid.is_dir(): + raise FileNotFoundError(f"Lance_3B_Video not found under {snapshot}") + return str(img), str(vid) + + +def _init_models( + snapshot: str, + ulysses_degree: int = 1, + replicas_per_omni: int = 1, +): + """Load the two Lance checkpoints onto distinct GPU sets so image + and video tasks can run truly in parallel, optionally with Ulysses + sequence parallel inside each Omni. + + Device layout (with ``CUDA_VISIBLE_DEVICES=$DEVICES``): + + - ``ulysses_degree=1`` (P0): image Omni → logical 0, video Omni → + logical 1. Concurrent throughput = 2x serial; needs 2 GPUs total. + - ``ulysses_degree=2`` (P1): image Omni → logical ``0,1``, video + Omni → logical ``2,3``. Each Omni internally splits the DiT + denoising sequence across 2 GPUs (Ulysses SP). Total = 4 GPUs; + single-request t2v/i2v ~1.5-1.8x faster *and* image+video still + concurrent. + - ``ulysses_degree=N``: image → ``0..N-1``, video → ``N..2N-1``. + Needs ``2*N`` GPUs. + + Falls back gracefully when there aren't enough GPUs: prints a + warning and downgrades to the largest feasible degree. + """ + global _OMNI_IMAGE, _OMNI_VIDEO, _PARAMS_IMAGE, _PARAMS_VIDEO + from vllm_omni.entrypoints.omni import Omni + + img_ckpt, vid_ckpt = _resolve_ckpts(snapshot) + + import os + + cvd = os.environ.get("CUDA_VISIBLE_DEVICES", "") + visible = [d for d in cvd.split(",") if d.strip()] if cvd else [] + n_visible = len(visible) + + # Resolve ulysses & replicas against available device count. + if ulysses_degree < 1: + ulysses_degree = 1 + if replicas_per_omni < 1: + replicas_per_omni = 1 + # Per-Omni GPU count = ulysses_degree × replicas_per_omni + # (ulysses splits a single replica across GPUs; replicas multiply by GPU). + per_omni_gpus = ulysses_degree * replicas_per_omni + needed = 2 * per_omni_gpus + if needed > max(n_visible, 1): + # Downgrade replicas first (cheaper than losing SP), then ulysses. + feasible_replicas = max(1, n_visible // (2 * ulysses_degree)) + if feasible_replicas < replicas_per_omni: + print( + f"[unified] WARN: --replicas-per-omni={replicas_per_omni} × " + f"ulysses_degree={ulysses_degree} needs {needed} GPUs but only " + f"{n_visible} are visible. Falling back to replicas={feasible_replicas}.", + flush=True, + ) + replicas_per_omni = feasible_replicas + per_omni_gpus = ulysses_degree * replicas_per_omni + if 2 * per_omni_gpus > n_visible: + feasible_sp = max(1, n_visible // 2) + if ulysses_degree > feasible_sp: + print( + f"[unified] WARN: --ulysses-degree={ulysses_degree} needs " + f"{2 * ulysses_degree} GPUs. Falling back to ulysses_degree={feasible_sp}.", + flush=True, + ) + ulysses_degree = feasible_sp + per_omni_gpus = ulysses_degree * replicas_per_omni + + can_split = n_visible >= 2 + use_sp = ulysses_degree > 1 + use_replicas = replicas_per_omni > 1 + + print( + f"[unified] CUDA_VISIBLE_DEVICES={cvd!r} n_visible={n_visible} " + f"can_split={can_split} ulysses_degree={ulysses_degree} " + f"replicas_per_omni={replicas_per_omni} per_omni_gpus={per_omni_gpus}", + flush=True, + ) + + # Build per-Omni device strings. Image takes the lower half, video + # the upper half. + if per_omni_gpus > 1: + img_devices = ",".join(str(i) for i in range(per_omni_gpus)) + vid_devices = ",".join(str(i) for i in range(per_omni_gpus, 2 * per_omni_gpus)) + elif can_split: + img_devices, vid_devices = "0", "1" + else: + img_devices, vid_devices = "0", "0" + + print( + f"[unified] Loading Lance_3B (image weights) on logical devices {img_devices!r}" + f" replicas={replicas_per_omni}...", + flush=True, + ) + # Lance single-stage engine defaults — mirrors the offline ``end2end.py`` + # flat-kwargs path so this demo does not depend on ``deploy/lance.yaml``. + # ``pipeline="lance"`` is the explicit selector (Lance HF config has no + # ``model_type`` field, so auto-detect from the model dir alone fails). + lance_engine_defaults = dict( + pipeline="lance", + max_num_batched_tokens=32768, + max_num_seqs=1, + enforce_eager=True, + trust_remote_code=True, + enable_prefix_caching=False, + async_chunk=False, + ) + + img_kwargs: dict = {"model": img_ckpt, **lance_engine_defaults} + if img_devices != "0": + img_kwargs["stage_0_devices"] = img_devices + if use_sp: + # Per the post-#3483 API, parallel knobs flow through *flat* + # ``ulysses_degree`` / ``ring_degree`` kwargs on the Omni + # constructor; ``_apply_diffusion_parallel_runtime_overrides`` + # moves them into a nested ``parallel_config`` dict for the stage. + img_kwargs["ulysses_degree"] = ulysses_degree + if use_replicas: + img_kwargs["stage_0_num_replicas"] = replicas_per_omni + _OMNI_IMAGE = Omni(**img_kwargs) + _PARAMS_IMAGE = _OMNI_IMAGE.default_sampling_params_list + + print( + f"[unified] Loading Lance_3B_Video (video weights) on logical devices {vid_devices!r}" + f" replicas={replicas_per_omni}...", + flush=True, + ) + vid_kwargs: dict = {"model": vid_ckpt, **lance_engine_defaults} + vid_kwargs["stage_0_devices"] = vid_devices + if use_sp: + vid_kwargs["ulysses_degree"] = ulysses_degree + if use_replicas: + vid_kwargs["stage_0_num_replicas"] = replicas_per_omni + _OMNI_VIDEO = Omni(**vid_kwargs) + _PARAMS_VIDEO = _OMNI_VIDEO.default_sampling_params_list + print("[unified] Both models ready.", flush=True) + + +def _omni_for(task: str): + if task in {"t2v", "i2v", "video_edit", "x2t_video"}: + return _OMNI_VIDEO, _PARAMS_VIDEO + return _OMNI_IMAGE, _PARAMS_IMAGE + + +# ------------------------------------------ size resolution helpers ----- + +_AR_RATIOS: dict[str, float] = { + # Mirror upstream's 6 buckets exactly — *no* 3:2 / 2:3, those are + # snapped to 4:3 / 3:4 by upstream's bucket selector. Must match + # ``ASPECT_RATIO_CHOICES`` or gradio will reject the auto-picked + # value as "not in the list of choices". + "21:9": 21 / 9, + "16:9": 16 / 9, + "4:3": 4 / 3, + "1:1": 1.0, + "3:4": 3 / 4, + "9:16": 9 / 16, +} + + +def _closest_aspect_ratio(width: int, height: int) -> str: + """Snap an arbitrary ``W×H`` to the nearest preset in + ``ASPECT_RATIO_CHOICES`` (used to auto-select the dropdown when the + user uploads / clicks an i2v example image). Distance is computed + in log-space so 4:3 isn't unfairly favored over 1:1 just because of + linear-scale proximity.""" + import math + + if not width or not height: + return DEFAULT_VIDEO_AR + img_log = math.log(width / height) + best = min(_AR_RATIOS.items(), key=lambda kv: abs(math.log(kv[1]) - img_log)) + return best[0] + + +def get_size_for_task(task: str, ar: str, resolution: str): + """Return (width, height) for a task + aspect-ratio + resolution preset.""" + if task in {"t2v", "i2v", "video_edit"}: + m = VIDEO_RES_MAP.get(resolution, VIDEO_480P_AR_TO_SIZE) + return m.get(ar, m["16:9"]) + elif task in {"t2i", "image_edit"}: + return IMAGE_AR_TO_SIZE.get(ar, IMAGE_AR_TO_SIZE["1:1"]) + return (640, 352) + + +def _format_size_md(task: str, w: int, h: int) -> str: + unit = "frames" if task in {"t2v", "i2v", "video_edit"} else "px" + return f"{w} × {h} {unit}" + + +# ------------------------------------------ per-task prompt formats ----- + + +def _vision_block(kind: str = "video"): + from vllm_omni.diffusion.models.lance.prompts import ( + IMAGE_PAD, + VIDEO_PAD, + VISION_END, + VISION_START, + ) + + pad = VIDEO_PAD if kind == "video" else IMAGE_PAD + return f"{VISION_START}{pad}{VISION_END}" + + +def _format(task, prompt, image, video, num_frames, height, width, cfg, ts_shift): + from vllm_omni.diffusion.models.lance.prompts import render_lance_prompt + + if task == "t2i": + return [ + { + "prompt": render_lance_prompt("t2i", prompt), + "modalities": ["image"], + "extra_args": {"height": int(height), "width": int(width), "cfg_text_scale": float(cfg)}, + } + ] + if task == "image_edit": + return [ + { + "prompt": render_lance_prompt("image_edit", prompt, vision_token=_vision_block("image")), + "multi_modal_data": {"img2img": image}, + "modalities": ["image"], + "extra_args": {"cfg_text_scale": float(cfg)}, + } + ] + if task == "x2t_image": + q = (prompt or "").strip() or "Describe this image in detail." + # Upstream's official x2t_image examples carry a QA-style + # per-example system prompt ("Look at the image carefully and + # answer the question."). Without it, Lance's caption-default + # system prompt makes the model describe the image instead of + # answering — matching that here so gradio out-of-the-box agrees + # with the upstream CLI on VQA cases. + return [ + { + "prompt": render_lance_prompt( + "x2t_image", + q, + vision_token=_vision_block("image"), + system_prompt="Look at the image carefully and answer the question.", + ), + "multi_modal_data": {"image": image}, + "modalities": ["text"], + "extra_args": {"max_think_tokens": 500}, + } + ] + if task == "t2v": + return [ + { + "prompt": render_lance_prompt("t2v", prompt), + "modalities": ["video"], + "extra_args": { + "num_frames": int(num_frames), + "video_height": int(height), + "video_width": int(width), + "cfg_text_scale": float(cfg), + "timestep_shift": float(ts_shift), + }, + } + ] + if task == "i2v": + return [ + { + "prompt": render_lance_prompt("i2v", prompt, vision_token=_vision_block("video")), + "multi_modal_data": {"first_frame": image}, + "modalities": ["video"], + "extra_args": { + "num_frames": int(num_frames), + "video_height": int(height), + "video_width": int(width), + "cfg_text_scale": float(cfg), + "timestep_shift": float(ts_shift), + }, + } + ] + if task == "video_edit": + return [ + { + "prompt": render_lance_prompt("video_edit", prompt, vision_token=_vision_block("video")), + "multi_modal_data": {"video": video}, + "modalities": ["video"], + "extra_args": { + "num_frames": int(num_frames), + "video_height": int(height), + "video_width": int(width), + "cfg_text_scale": float(cfg), + "timestep_shift": float(ts_shift), + }, + } + ] + if task == "x2t_video": + q = (prompt or "").strip() or "Describe this video in detail." + return [ + { + "prompt": render_lance_prompt( + "x2t_video", + q, + vision_token=_vision_block("video"), + system_prompt="Watch the video carefully and answer the question.", + ), + "multi_modal_data": {"video": video}, + "modalities": ["text"], + "extra_args": {"max_think_tokens": 500}, + } + ] + raise ValueError(f"Unknown task: {task}") + + +def _extract_demo_sources(root: str, cache_dir: str = "/tmp/lance_demo_sources") -> list[tuple[str, str]]: + """Split upstream Lance's ``video-editing-demo-*.mp4`` and the + ``multi-turn-editing-demo-01.mp4`` filmstrip into runnable source + videos (the *input* half of each side-by-side demo) so they can be + surfaced as clickable ``gr.Examples`` for ``video_edit``. + + Upstream's 8 video-editing demos are **horizontally split** + ``(source | edit)`` — left half is the input video, right half is + the edited result. (A naïve top↔bottom RGB diff comparison is + misleading: the demos are landscape scenes where natural sky/ground + variance dominates the metric. Confirmed visually that the split + runs vertically through the middle.) The multi-turn-editing demo + is a horizontal filmstrip ``(panel0 | panel1 | … | panelN)`` where + panel 0 is the source and each subsequent panel adds one more edit + step. + + Returns ``[(source_video_path, suggested_prompt), ...]``. Generated + files are cached under ``cache_dir``; if the cache already has them + we skip re-encoding. + """ + from pathlib import Path as _Path + + import imageio.v3 as _iio + import numpy as _np + + if not root or not _Path(root).is_dir(): + return [] + root_p = _Path(root) + cache_p = _Path(cache_dir) + cache_p.mkdir(parents=True, exist_ok=True) + out: list[tuple[str, str]] = [] + + # Hand-curated prompts inferred by visually diffing each demo's left + # (source) vs right (edit) half — upstream's README ships the result + # videos but doesn't publish the prompts that produced them, so these + # are best-effort descriptions of the visible transformation. + demo_prompts = { + 1: "Add a roaring bonfire with bright orange flames behind the bear.", + 2: "Change the lush green tropical forest to a dry autumn forest with warm sunlight.", + 3: "Add several colorful balloons floating around the giraffe.", + 4: "Change her hair from red to natural chestnut brown and have her gently touch it with one hand.", + 5: "Replace the dog with a tabby cat, keeping the same red collar and car window.", + 6: "Remove the black square patch from her cheek.", + 7: "Change the boy to a girl with long black hair, blue eyes, and a black dress.", + 8: "Place her in a snowy forest and add a red striped scarf around her neck.", + } + + # ---- 8 horizontally-split video-editing demos ---- + for i in range(1, 9): + src_demo = root_p / "assets" / "video-editing" / "videos" / f"video-editing-demo-{i:02d}.mp4" + if not src_demo.exists(): + continue + out_path = cache_p / f"video-edit-demo-{i:02d}-source.mp4" + if not out_path.exists(): + arr = _iio.imread(str(src_demo), plugin="pyav") + # Horizontal split: take left half (width axis). + left = arr[:, :, : arr.shape[2] // 2, :] + left = _np.ascontiguousarray(left) + _iio.imwrite(str(out_path), left, fps=12, codec="libx264", quality=8) + out.append((str(out_path), demo_prompts.get(i, "Describe the edit you'd like to apply."))) + + # ---- multi-turn editing filmstrip → first panel as source ---- + mt_src = root_p / "assets" / "multi-turn-editing" / "videos" / "multi-turn-editing-demo-01.mp4" + if mt_src.exists(): + out_path = cache_p / "multi-turn-edit-source.mp4" + if not out_path.exists(): + arr = _iio.imread(str(mt_src), plugin="pyav") + T, H, W, _C = arr.shape + # The filmstrip is roughly ``N`` square-ish panels of width H + # each; pick the first panel as the source. W/H ≈ 11.7 on the + # shipped clip → panel_width ≈ H to keep aspect. + panel_w = H + first = arr[:, :, :panel_w, :] + first = _np.ascontiguousarray(first) + _iio.imwrite(str(out_path), first, fps=12, codec="libx264", quality=8) + out.append( + ( + str(out_path), + "Apply a multi-turn consistency edit — try sequential edits like " + "'change clothing color, then change background, then add accessories'.", + ) + ) + + return out + + +def _load_lance_showcase(root: str) -> dict[str, list[str]]: + """Collect upstream Lance's README showcase videos as visual + reference (full side-by-side ``before|after`` demos). Distinct from + :func:`_extract_demo_sources`, which crops just the source half so + they can be run. + + Surfaces: + * ``video_edit``: 8 ``video-editing-demo-{01..08}.mp4`` + the + ``multi-turn-editing-demo-01.mp4`` filmstrip. + """ + from pathlib import Path as _Path + + if not root or not _Path(root).is_dir(): + return {} + root_p = _Path(root) + out: dict[str, list[str]] = {} + + edit_videos: list[str] = [] + for i in range(1, 9): + p = root_p / "assets" / "video-editing" / "videos" / f"video-editing-demo-{i:02d}.mp4" + if p.exists(): + edit_videos.append(str(p)) + mt = root_p / "assets" / "multi-turn-editing" / "videos" / "multi-turn-editing-demo-01.mp4" + if mt.exists(): + edit_videos.append(str(mt)) + if edit_videos: + out["video_edit"] = edit_videos + + return out + + +def _load_lance_examples(root: str) -> dict[str, list[list]]: + """Parse upstream Lance's ``config/examples/*.json`` into clickable + presets keyed by task name. + + Each value is the list of rows that :class:`gr.Examples` expects, with + one row per official sample. The row column order matches the + ``inputs`` we'll bind below: ``[prompt, image_path | None, video_path | None]``. + + Path resolution: example JSONs reference media via repo-relative paths + (``config/examples/...``). We join against ``root`` so the gradio + process can serve the files directly. Examples whose media is + missing on disk are skipped silently so a partial upstream checkout + still works. + """ + import json as _json + from pathlib import Path as _Path + + if not root or not _Path(root).is_dir(): + return {} + root_p = _Path(root) + out: dict[str, list[list]] = {} + + def _resolve(rel: str) -> str | None: + if not isinstance(rel, str): + return None + p = root_p / rel + return str(p) if p.exists() else None + + def _safe_json(name: str): + f = root_p / "config" / "examples" / name + if not f.exists(): + return None + try: + return _json.loads(f.read_text()) + except Exception: # malformed json → skip + return None + + # ---- t2i / t2v: ``{filename: prompt}`` maps; only prompt is useful as + # an input preset (the filename is the *expected output*, not an + # input). ---- + for task, fname in (("t2i", "t2i_example.json"), ("t2v", "t2v_example.json")): + data = _safe_json(fname) + if isinstance(data, dict): + out[task] = [[prompt, None, None] for _, prompt in data.items() if isinstance(prompt, str)] + + # ---- i2v / video_edit: ``interleave_array = [prompt, media_path]`` ---- + # NOTE: image_edit handled separately below because upstream ships + # only one source image with 5 different prompts; we substitute + # diverse source images so each gradio example is visually distinct. + for task, fname, modality in ( + ("i2v", "i2v_example.json", "image"), + ("video_edit", "video_edit_example.json", "video"), + ): + data = _safe_json(fname) + if not isinstance(data, dict): + continue + rows: list[list] = [] + for _, sample in data.items(): + ia = sample.get("interleave_array") if isinstance(sample, dict) else None + if not (isinstance(ia, list) and len(ia) >= 2): + continue + prompt, media = ia[0], ia[1] + media_path = _resolve(media) + if media_path is None: + continue + row = [prompt, media_path, None] if modality == "image" else [prompt, None, media_path] + rows.append(row) + if rows: + out[task] = rows + + # ---- image_edit: curated visually-distinct pairs ---- + # Upstream ``image_edit_example.json`` ships 5 *different prompts* against + # the *same* source image (``edit_img.jpg``), which makes for a confusing + # gradio gallery (5 thumbnails that look identical). We substitute 5 + # visually-distinct sources from the i2v frame pool — same upstream + # repo, just paired differently — and write a fresh edit prompt for + # each so users see what kinds of transforms image_edit supports. + image_edit_pairs = [ + # (relative source path, prompt) + ("config/examples/image_edit_examples/edit_img.jpg", "Remove the hat from the painting."), + ( + "config/examples/text_image_to_video_examples/00001.png", + "Replace the icy glacier background with a sandy desert.", + ), + ( + "config/examples/text_image_to_video_examples/00005.png", + "Change her hair to bright red curly hair and add red lipstick.", + ), + ("config/examples/text_image_to_video_examples/00004.png", "Convert the scene into a 3D cartoon render style."), + ( + "config/examples/text_image_to_video_examples/00003.png", + "Add a vivid sunset sky with orange and purple clouds behind the penguin.", + ), + ] + ie_rows: list[list] = [] + for rel, prompt in image_edit_pairs: + path = _resolve(rel) + if path is not None: + ie_rows.append([prompt, path, None]) + if ie_rows: + out["image_edit"] = ie_rows + + # ---- Append the cropped video-editing showcase demos to ``video_edit`` ---- + # ``_extract_demo_sources`` slices the *source* half out of upstream's + # vertically-stacked side-by-side clips so they're runnable as + # ``video_edit`` examples (the bare side-by-side mp4s have the edit + # output baked in and would re-edit the wrong frames). + demo_rows = _extract_demo_sources(root) + if demo_rows: + existing = out.get("video_edit", []) + for src_path, prompt in demo_rows: + existing.append([prompt, None, src_path]) + out["video_edit"] = existing + + # ---- x2t_image / x2t_video: ``interleave_array = [media_path, [system, question, answer]]`` ---- + for task, fname, modality in ( + ("x2t_image", "x2t_image_example.json", "image"), + ("x2t_video", "x2t_video_example.json", "video"), + ): + data = _safe_json(fname) + if not isinstance(data, dict): + continue + rows: list[list] = [] + for _, sample in data.items(): + ia = sample.get("interleave_array") if isinstance(sample, dict) else None + if not (isinstance(ia, list) and len(ia) >= 2 and isinstance(ia[1], list) and len(ia[1]) >= 2): + continue + media_path = _resolve(ia[0]) + question = ia[1][1] + if media_path is None or not isinstance(question, str): + continue + row = [question, media_path, None] if modality == "image" else [question, None, media_path] + rows.append(row) + if rows: + out[task] = rows + + return out + + +def _to_pil(x): + if x is None: + return None + if isinstance(x, Image.Image): + return x.convert("RGB") + if isinstance(x, np.ndarray): + return Image.fromarray(x).convert("RGB") + if isinstance(x, str): + return Image.open(x).convert("RGB") + raise ValueError(f"Unsupported image type {type(x)}") + + +def _save_video(frames): + arr = np.stack([np.asarray(f) for f in frames], axis=0).astype(np.uint8) + out = Path(tempfile.mkdtemp(prefix="lance_grad_")) / "out.mp4" + iio.imwrite(str(out), arr, fps=12, codec="libx264", quality=8) + return str(out) + + +def _save_image(img): + out = Path(tempfile.mkdtemp(prefix="lance_grad_")) / "out.png" + if isinstance(img, np.ndarray): + Image.fromarray(img).save(out) + elif isinstance(img, Image.Image): + img.save(out) + else: + raise ValueError(type(img)) + return str(out) + + +# ---------------------------------------------------- generation ----- + + +def run_task( + task_label, + prompt, + image, + video, + aspect_ratio, + resolution, + video_duration_sec, + steps, + cfg, + ts_shift, + seed, + num_frames_override, +): + task = LABEL_TO_TASK[task_label] + omni, params = _omni_for(task) + + image = _to_pil(image) if image is not None else None + + # Resolve W/H from aspect + resolution + width, height = get_size_for_task(task, aspect_ratio, resolution) + + # Frames: for video tasks, use duration × fps + if task in {"t2v", "i2v", "video_edit"}: + fps = 12 + if num_frames_override and num_frames_override > 0: + num_frames = int(num_frames_override) + else: + num_frames = int(video_duration_sec * fps + 1) # +1 for first frame + else: + num_frames = 1 + + sp = params[0] + if seed is not None: + try: + sp.seed = int(seed) + except Exception: + sp.seed = DEFAULT_SEED + sp.num_inference_steps = int(steps) + + formatted = _format(task, prompt or "", image, video, num_frames, height, width, cfg, ts_shift) + print(f"[unified] task={task} {width}x{height} frames={num_frames} cfg={cfg} steps={steps} seed={seed}", flush=True) + + outputs = list(omni.generate(prompts=formatted, sampling_params_list=params)) + custom = getattr(outputs[0], "custom_output", None) or {} + # Lance/Bagel return text at outputs[0].output (string) or + # custom_output["text_output"]. + text_out = None + if task in {"x2t_image", "x2t_video"}: + text_out = custom.get("text_output") or getattr(outputs[0], "output", None) + if not isinstance(text_out, str): + text_out = None + + # NOTE: previously this returned a 4-tuple ending in a status string + # rendered into a ``gr.Markdown`` slot below the outputs. That slot + # left an empty grey bar in the UI before the first run and was + # dropped — Gradio's built-in error toast already surfaces failures, + # and successful runs are self-evident from the output panel. + if task in {"x2t_image", "x2t_video"}: + return ( + gr.update(visible=False), + gr.update(visible=False), + gr.update(value=str(text_out or "(empty)"), visible=True), + ) + if task in {"t2v", "i2v", "video_edit"}: + frames = custom.get("video_frames") + if frames is None: + raise gr.Error("No video frames in output.") + return (gr.update(visible=False), gr.update(value=_save_video(frames), visible=True), gr.update(visible=False)) + # image tasks — orchestrator strips ``outputs[0].output`` across the + # IPC boundary but preserves ``custom_output``, so prefer the latter. + img_out = custom.get("image") or getattr(outputs[0], "output", None) + if img_out is None: + raise gr.Error("No image in output.") + return (gr.update(value=_save_image(img_out), visible=True), gr.update(visible=False), gr.update(visible=False)) + + +# -------------------------------------------------- UI building ----- + +# Custom CSS — minimal, matches Lance branding (rounded panels + dark header). +APP_CSS = """ +.lance-hero { + display: flex; flex-direction: column; align-items: center; + gap: 12px; padding: 18px 12px; margin-bottom: 8px; +} +.lance-hero .logos { + display: flex; align-items: center; gap: 24px; justify-content: center; +} +.lance-hero .logos img { height: 64px; } +.lance-hero .logos .x { font-size: 28px; color: #888; font-weight: 300; } +.lance-hero .vllm-logo { height: 44px; } +.lance-hero .lance-title { + margin: 6px 0 0 0; font-size: 1.45rem; font-weight: 600; + text-align: center; color: #475569; +} +.lance-hero .lance-badges { display: flex; gap: 8px; flex-wrap: wrap; justify-content: center; } +.lance-panel { padding: 12px; border-radius: 12px; } +.task-selector .wrap { display: flex; flex-wrap: wrap; gap: 6px; } +.lance-run-button { font-size: 1.05rem !important; padding: 12px 18px !important; } +/* Reserve a single line of vertical space so the layout doesn't jump + when status text appears/disappears — but ``:empty`` collapses to 0 + so a placeholder ``gr.Markdown("")`` (e.g. the run-status slot before + the first generation) doesn't render as a phantom grey bar. */ +.lance-run-status { text-align: center; color: #475569; min-height: 1.4em; } +.lance-run-status:empty, +.lance-run-status .prose:empty, +.lance-run-status .prose p:empty { min-height: 0 !important; padding: 0 !important; margin: 0 !important; } +.lance-display-frame video, .lance-display-frame img { max-height: 560px; } +""" + + +def _embed_logo(path: str, mime: str) -> str: + if not os.path.exists(path): + return "" + with open(path, "rb") as f: + b64 = base64.b64encode(f.read()).decode("ascii") + return f"data:{mime};base64,{b64}" + + +def build_header_html() -> str: + lance_uri = _embed_logo("/home/zjy/code/lance-upstream/assets/logo/lance-logo.webp", "image/webp") + vllm_uri = _embed_logo("/home/zjy/code/vllm-omni/docs/source/logos/vllm-omni-logo.png", "image/png") + logos_html = "" + if lance_uri: + logos_html += f'Lance' + if vllm_uri: + logos_html += '×' + logos_html += f'' + return f""" +
+
{logos_html}
+

Lance × vllm-omni · Unified Multimodal Inference

+
+ Homepage + Paper + HF + Lance GitHub + vllm-omni GitHub +
+
+ """ + + +def build_ui( + examples_by_task: dict[str, list[list]] | None = None, + showcase_by_task: dict[str, list[str]] | None = None, +): + examples_by_task = examples_by_task or {} + showcase_by_task = showcase_by_task or {} + with gr.Blocks(title="Lance × vllm-omni", css=APP_CSS) as demo: + gr.HTML(build_header_html()) + + # ── Task selector ── + task_radio = gr.Radio( + label="Task", + show_label=False, + choices=list(LABEL_TO_TASK.keys()), + value=TASK_I2V_LABEL, + elem_classes=["task-selector"], + ) + + with gr.Row(): + # ─── LEFT: inputs ─── + with gr.Column(scale=1): + with gr.Group(elem_classes=["lance-panel"]): + prompt_in = gr.Textbox( + label="Prompt", + lines=5, + placeholder="Describe the desired output...", + ) + # Aspect ratio is **not user-configurable** — upstream + # Lance always snaps to one of 6 hard-coded buckets based + # on the input image / video. We keep the component in + # the tree (so existing event wiring still works) but + # render it invisibly; ``_auto_aspect_from_image`` / + # ``_auto_aspect_from_video`` are the only writers. + aspect_in = gr.Dropdown( + label="Aspect Ratio (auto)", + choices=ASPECT_RATIO_CHOICES, + value=DEFAULT_VIDEO_AR, + visible=False, + interactive=False, + ) + with gr.Row(): + resolution_in = gr.Dropdown( + label="Resolution", + choices=VIDEO_RES_CHOICES, + value=DEFAULT_VIDEO_RES, + visible=True, + ) + duration_in = gr.Slider( + 1, + MAX_VIDEO_DURATION_SEC, + value=DEFAULT_VIDEO_DURATION_SEC, + step=1, + label="Duration (sec)", + ) + size_md = gr.Markdown(_format_size_md("i2v", 848, 480), elem_classes=["lance-run-status"]) + + image_in = gr.Image(label="Input Image", type="pil", visible=True, elem_classes=["lance-display-frame"]) + video_in = gr.Video(label="Input Video", visible=False, elem_classes=["lance-display-frame"]) + + with gr.Accordion("Advanced Parameters", open=False): + seed_in = gr.Number(label="Seed", value=DEFAULT_SEED, precision=0) + steps_in = gr.Slider(5, 80, value=DEFAULT_TIMESTEPS, step=5, label="Denoise steps") + with gr.Row(): + cfg_in = gr.Number(label="CFG text scale", value=DEFAULT_CFG) + ts_shift_in = gr.Number(label="Timestep shift", value=DEFAULT_TIMESTEP_SHIFT) + num_frames_override = gr.Number( + label="Override num_frames (0 = derive from duration)", + value=0, + precision=0, + ) + + run_btn = gr.Button("🚀 Generate", variant="primary", elem_classes=["lance-run-button"]) + + # ─── RIGHT: output ─── + with gr.Column(scale=1): + with gr.Group(elem_classes=["lance-panel"]): + image_out = gr.Image(label="Output Image", visible=False, elem_classes=["lance-display-frame"]) + video_out = gr.Video( + label="Output Video", + autoplay=True, + loop=True, + visible=True, + elem_classes=["lance-display-frame"], + ) + text_out = gr.Textbox(label="Output Caption", lines=6, visible=False) + # Status text was previously rendered as ``gr.Markdown("", + # ...)`` which left an empty grey bar in the UI before the + # first run. Dropped — Gradio's built-in toast / error + # bubble already surfaces success / failure, so the extra + # placeholder added no signal but stole vertical real estate. + + # ─── Official Lance examples per task ─── + # Click an example row → prompt + image/video auto-fill. Each + # task has its own ``gr.Examples`` block; only the active task's + # block is shown. Order matches LABEL_TO_TASK (radio order). + _initial_task = LABEL_TO_TASK[TASK_I2V_LABEL] # default selection + _example_groups: dict[str, gr.Group] = {} + if examples_by_task: + # Per-task input bindings — Gradio's ``gr.Video`` doesn't reliably + # update from a programmatic example click when the Examples row + # also includes ``None`` placeholders for ``Image`` (the player + # stays blank). Binding each task's Examples block to *only* the + # components it actually fills sidesteps that issue. + task_input_components: dict[str, list] = { + "t2i": [prompt_in], + "t2v": [prompt_in], + "i2v": [prompt_in, image_in], + "image_edit": [prompt_in, image_in], + "x2t_image": [prompt_in, image_in], + "video_edit": [prompt_in, video_in], + "x2t_video": [prompt_in, video_in], + } + + def _project_row(task: str, row: list) -> list: + # ``examples_by_task`` rows are always ``[prompt, image, video]`` + # (3 columns). Project down to whatever this task binds. + prompt, img, vid = row + comps = task_input_components[task] + out = [] + for c in comps: + if c is prompt_in: + out.append(prompt) + elif c is image_in: + out.append(img) + elif c is video_in: + out.append(vid) + return out + + for task in ("t2i", "t2v", "i2v", "image_edit", "video_edit", "x2t_image", "x2t_video"): + rows = examples_by_task.get(task) + if not rows: + continue + projected_rows = [_project_row(task, r) for r in rows] + with gr.Group(visible=(task == _initial_task), elem_classes=["lance-panel"]) as grp: + gr.Markdown(f"### 📁 Official Lance examples — `{task}` (click to load)") + # Bind to only the components this task actually populates. + gr.Examples( + examples=projected_rows, + inputs=task_input_components[task], + label="", + examples_per_page=4, + cache_examples=False, + ) + # NOTE: Upstream's 8 ``video-editing-demo-*.mp4`` + the + # ``multi-turn-editing-demo-01.mp4`` filmstrip are merged + # into the runnable examples list above via + # ``_extract_demo_sources`` (left-half / first-panel crops + # become the source video, paired with a generic editable + # prompt). No separate showcase gallery is needed. + _example_groups[task] = grp + + gr.Markdown("""--- +💡 **Tips:** +* **Video Generation**: text → 5-second video at 480p +* **Image to Video**: upload first frame + describe motion; for subtle facial expressions try CFG 10-15 +* **Image Edit**: upload image + describe the edit; outputs a new image +* **Video Edit**: upload video + describe the edit; outputs a modified video +* **Understanding tasks**: upload media; outputs caption text""") + + # ─────────────── UI logic ─────────────── + + # Fixed task order for the example-group visibility tail of on_task's + # outputs — used both to build the return list and to register the + # corresponding output components below. Must stay in sync with the + # group-creation loop above. + _example_task_order = [ + t + for t in ("t2i", "t2v", "i2v", "image_edit", "video_edit", "x2t_image", "x2t_video") + if t in _example_groups + ] + + def on_task(label): + task = LABEL_TO_TASK[label] + is_video_task = task in {"t2v", "i2v", "video_edit"} + is_image_task = task in {"t2i", "image_edit"} + is_understanding = task in {"x2t_image", "x2t_video"} + # Understanding tasks don't need a prompt — system prompt handles + # captioning. Generation/edit tasks need a prompt. + need_text = not is_understanding + need_image = task in {"i2v", "image_edit", "x2t_image"} + need_video = task in {"video_edit", "x2t_video"} + default_ar = DEFAULT_VIDEO_AR if (is_video_task or is_understanding) else DEFAULT_IMAGE_AR + w, h = get_size_for_task(task, default_ar, DEFAULT_VIDEO_RES) + base = [ + gr.update(visible=need_text), # prompt_in + gr.update(visible=need_image, value=None), # image_in + gr.update(visible=need_video, value=None), # video_in + gr.update(value=default_ar, visible=False), # aspect (always hidden, auto-detected) + gr.update(visible=is_video_task, value=DEFAULT_VIDEO_RES), # resolution + gr.update( + visible=is_video_task, + value=DURATION_DEFAULT_PER_TASK.get(task, DEFAULT_VIDEO_DURATION_SEC), + ), # duration — matches upstream's per-script num_frames default + gr.update(value=_format_size_md(task, w, h)), # size_md + gr.update(visible=is_image_task and not is_video_task), # image_out (only show for image-gen tasks) + gr.update(visible=is_video_task), # video_out + gr.update(visible=is_understanding), # text_out + ] + return base + [gr.update(visible=(t == task)) for t in _example_task_order] + + task_radio.change( + on_task, + [task_radio], + [ + prompt_in, + image_in, + video_in, + aspect_in, + resolution_in, + duration_in, + size_md, + image_out, + video_out, + text_out, + ] + + [_example_groups[t] for t in _example_task_order], + ) + + def on_aspect_or_res(task_label, ar, resolution): + task = LABEL_TO_TASK[task_label] + w, h = get_size_for_task(task, ar, resolution) + return gr.update(value=_format_size_md(task, w, h)) + + aspect_in.change(on_aspect_or_res, [task_radio, aspect_in, resolution_in], [size_md]) + resolution_in.change(on_aspect_or_res, [task_radio, aspect_in, resolution_in], [size_md]) + + # ── Auto-pick aspect ratio from the input media ── + # For tasks where the model is conditioned on an input + # image/video (image_edit, i2v, video_edit), the output should + # preserve the source aspect ratio — otherwise the model has to + # squeeze / stretch the scene to fill a different frame, and the + # geometry drifts (e.g. the snow-leopard "stepping into the + # void" case: a 3:2 input rendered into a 16:9 output pushed the + # landing platform off-frame). When the user uploads or clicks + # an example, snap the aspect dropdown to whichever preset is + # closest to the source. + def _auto_aspect_from_image(task_label, img): + task = LABEL_TO_TASK[task_label] + if task not in {"image_edit", "i2v"} or img is None: + return gr.update(), gr.update() + try: + pil = _to_pil(img) + w, h = pil.size + except Exception: + return gr.update(), gr.update() + ar = _closest_aspect_ratio(w, h) + new_w, new_h = get_size_for_task(task, ar, DEFAULT_VIDEO_RES) + return ( + gr.update(value=ar), + gr.update(value=_format_size_md(task, new_w, new_h)), + ) + + def _auto_aspect_from_video(task_label, vid_path): + task = LABEL_TO_TASK[task_label] + if task != "video_edit" or not vid_path: + return gr.update(), gr.update() + try: + import imageio.v3 as _iio + + meta = _iio.immeta(vid_path, plugin="pyav") + # ``immeta`` ships ``size = (w, h)`` for video readers. + if "size" in meta: + w, h = meta["size"] + else: + # Fall back to a single-frame read. + arr = _iio.imread(vid_path, plugin="pyav", index=0) + h, w = arr.shape[:2] + except Exception: + return gr.update(), gr.update() + ar = _closest_aspect_ratio(w, h) + new_w, new_h = get_size_for_task(task, ar, DEFAULT_VIDEO_RES) + return ( + gr.update(value=ar), + gr.update(value=_format_size_md(task, new_w, new_h)), + ) + + image_in.change( + _auto_aspect_from_image, + [task_radio, image_in], + [aspect_in, size_md], + ) + video_in.change( + _auto_aspect_from_video, + [task_radio, video_in], + [aspect_in, size_md], + ) + + run_btn.click( + run_task, + inputs=[ + task_radio, + prompt_in, + image_in, + video_in, + aspect_in, + resolution_in, + duration_in, + steps_in, + cfg_in, + ts_shift_in, + seed_in, + num_frames_override, + ], + outputs=[image_out, video_out, text_out], + ) + + return demo + + +def main(): + args = parse_args() + # Lance aligns better with upstream when using FLASH_ATTN + # (upstream uses flash_attn_varlen_func; SDPA accumulates ~5x more + # numerical drift through the 36-layer Qwen2 stack). Default to + # flash_attn unless the user explicitly set a backend. + os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN") + print("=" * 60, flush=True) + print("[unified] Lance × vllm-omni demo", flush=True) + print(f" snapshot: {args.model}", flush=True) + print(f" gradio: {args.host}:{args.port} (share={args.share})", flush=True) + print("=" * 60, flush=True) + _init_models( + args.model, + ulysses_degree=args.ulysses_degree, + replicas_per_omni=args.replicas_per_omni, + ) + examples_by_task = _load_lance_examples(args.examples_root) + if examples_by_task: + total = sum(len(rows) for rows in examples_by_task.values()) + per_task = ", ".join(f"{t}={len(r)}" for t, r in examples_by_task.items()) + print(f"[unified] Loaded {total} official examples ({per_task}) from {args.examples_root}", flush=True) + else: + print(f"[unified] No official examples loaded (examples_root={args.examples_root!r})", flush=True) + demo = build_ui(examples_by_task=examples_by_task) + # Concurrency budget = (image replicas + video replicas) so all the + # available GPU slots can run in parallel. P0 (1 replica each) = 2; + # ``--replicas-per-omni 4`` on 8 GPUs gives 8 concurrent slots. + queue_limit = 2 * max(1, args.replicas_per_omni) + print(f"[unified] Gradio queue concurrency limit = {queue_limit}", flush=True) + demo.queue(default_concurrency_limit=queue_limit) + # Whitelist the upstream checkout so gr.Examples can stage media that + # lives outside the gradio working dir / temp. Without this, clicks + # raise ``InvalidPathError`` and the example silently fails to load. + launch_kwargs: dict = dict(server_name=args.host, server_port=args.port, share=args.share) + allowed = [] + if args.examples_root and Path(args.examples_root).is_dir(): + allowed.append(str(Path(args.examples_root).resolve())) + # ``_extract_demo_sources`` caches cropped video-editing demo sources under + # ``/tmp/lance_demo_sources`` so they survive across launches; whitelist it + # too or the click-to-load will hit ``InvalidPathError``. + demo_cache = Path("/tmp/lance_demo_sources") + if demo_cache.is_dir(): + allowed.append(str(demo_cache.resolve())) + if allowed: + launch_kwargs["allowed_paths"] = allowed + demo.launch(**launch_kwargs) + + +if __name__ == "__main__": + main() diff --git a/examples/online_serving/lance/README.md b/examples/online_serving/lance/README.md new file mode 100644 index 000000000..5f2febce9 --- /dev/null +++ b/examples/online_serving/lance/README.md @@ -0,0 +1,40 @@ +# Lance — online serving + +OpenAI-compatible chat completions API for Lance, served by `vllm-omni`. + +## Start the server + +```bash +bash examples/online_serving/lance/run_server.sh +# or, with overrides +MODEL=bytedance-research/Lance \ +DEPLOY_CONFIG=vllm_omni/deploy/lance.yaml \ +PORT=8091 \ + bash examples/online_serving/lance/run_server.sh +``` + +The default deploy config is the single-stage `lance.yaml`; for `video_edit` +or `text2video` workloads, point ``DEPLOY_CONFIG`` at a video-checkpoint +config (or pass ``--model bytedance-research/Lance/Lance_3B_Video`` to +`run_server.sh`). + +## Send a request + +```bash +# text-to-image +python examples/online_serving/lance/openai_chat_client.py \ + --prompt "A cute corgi astronaut on the moon, cinematic" \ + --modality text2img \ + --output corgi.png + +# image edit +python examples/online_serving/lance/openai_chat_client.py \ + --prompt "Convert this into a vibrant cartoon-style illustration" \ + --modality img2img \ + --image-url path/to/photo.png \ + --output edited.png +``` + +The client is shared with BAGEL — same OpenAI message format, same +``modalities`` and ``num_inference_steps`` / ``seed`` / ``height`` / +``width`` knobs. diff --git a/examples/online_serving/lance/openai_chat_client.py b/examples/online_serving/lance/openai_chat_client.py new file mode 100755 index 000000000..1f470069b --- /dev/null +++ b/examples/online_serving/lance/openai_chat_client.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +""" +Lance OpenAI-compatible chat client for image generation and multimodal tasks. + +Usage: + python examples/online_serving/lance/openai_chat_client.py --prompt "A cute cat" --output output.png + python examples/online_serving/lance/openai_chat_client.py --prompt "Describe this image" --image-url https://example.com/image.png +""" + +import argparse +import base64 +from pathlib import Path + +import requests + + +def generate_image( + prompt: str, + server_url: str = "http://localhost:8091", + image_url: str | None = None, + height: int | None = None, + width: int | None = None, + steps: int | None = None, + seed: int | None = None, + negative_prompt: str | None = None, + modality: str = "text2img", # "text2img" (default), "img2img", "img2text", "text2text" +) -> bytes | str | None: + """Generate an image or text using the chat completions API. + + Args: + prompt: Text description or prompt + server_url: Server URL + image_url: URL or path to input image (for img2img/img2text) + height: Image height in pixels + width: Image width in pixels + steps: Number of inference steps + seed: Random seed + negative_prompt: Negative prompt + modality: Task modality hint + + Returns: + Image bytes (for image outputs) or Text string (for text outputs) or None if failed + """ + + # Construct Message Content + content = [{"type": "text", "text": f"<|im_start|>{prompt}<|im_end|>"}] + + if image_url: + # Check if local file + if Path(image_url).exists(): + with open(image_url, "rb") as f: + b64_data = base64.b64encode(f.read()).decode("utf-8") + final_image_url = f"data:image/jpeg;base64,{b64_data}" + else: + final_image_url = image_url + + content.append({"type": "image_url", "image_url": {"url": final_image_url}}) + + messages = [{"role": "user", "content": content}] + + # Build request payload with all parameters at top level + # Note: vLLM ignores "extra_body", so we put parameters directly in the payload + payload = {"messages": messages} + + # Set output modalities at top level + if modality == "text2img" or modality == "img2img": + payload["modalities"] = ["image"] + elif modality == "img2text" or modality == "text2text": + payload["modalities"] = ["text"] + + # Add generation parameters directly to payload + if height is not None: + payload["height"] = height + if width is not None: + payload["width"] = width + if steps is not None: + payload["num_inference_steps"] = steps + if seed is not None: + payload["seed"] = seed + if negative_prompt: + payload["negative_prompt"] = negative_prompt + + # Send request + try: + print(f"Sending request to {server_url} with modality {modality}...") + response = requests.post( + f"{server_url}/v1/chat/completions", + headers={"Content-Type": "application/json"}, + json=payload, + timeout=300, + ) + response.raise_for_status() + data = response.json() + + # Extract content - check ALL choices since server may return multiple + # (e.g., text in choices[0], image in choices[1]) + choices = data.get("choices", []) + + # First pass: look for image output in any choice + for choice in choices: + choice_content = choice.get("message", {}).get("content") + + # Handle Image Output + if isinstance(choice_content, list) and len(choice_content) > 0: + first_item = choice_content[0] + if isinstance(first_item, dict) and "image_url" in first_item: + img_url_str = first_item["image_url"].get("url", "") + if img_url_str.startswith("data:image"): + _, b64_data = img_url_str.split(",", 1) + return base64.b64decode(b64_data) + + # Second pass: look for text output if no image found + for choice in choices: + choice_content = choice.get("message", {}).get("content") + if isinstance(choice_content, str) and choice_content: + return choice_content + + print(f"Unexpected response format: {choices}") + return None + + except Exception as e: + print(f"Error: {e}") + return None + + +def main(): + parser = argparse.ArgumentParser(description="Bagel multimodal chat client") + parser.add_argument("--prompt", "-p", default="A cute cat", help="Text prompt") + parser.add_argument("--output", "-o", default="bagel_output.png", help="Output file (for image results)") + parser.add_argument("--server", "-s", default="http://localhost:8091", help="Server URL") + + # Modality Control + parser.add_argument("--image-url", "-i", type=str, help="Input image URL or local path") + parser.add_argument( + "--modality", + "-m", + default="text2img", + choices=["text2img", "img2img", "img2text", "text2text"], + help="Task modality", + ) + + # Generation Params + parser.add_argument("--height", type=int, default=512, help="Image height") + parser.add_argument("--width", type=int, default=512, help="Image width") + parser.add_argument("--steps", type=int, default=25, help="Inference steps") + parser.add_argument("--seed", type=int, default=42, help="Random seed") + parser.add_argument("--negative", help="Negative prompt") + + args = parser.parse_args() + + print(f"Mode: {args.modality}") + if args.image_url: + print(f"Input Image: {args.image_url}") + + result = generate_image( + prompt=args.prompt, + server_url=args.server, + image_url=args.image_url, + height=args.height, + width=args.width, + steps=args.steps, + seed=args.seed, + negative_prompt=args.negative, + modality=args.modality, + ) + + if result: + if isinstance(result, bytes): + # It's an image + output_path = Path(args.output) + output_path.write_bytes(result) + print(f"Image saved to: {output_path}") + print(f"Size: {len(result) / 1024:.1f} KB") + elif isinstance(result, str): + # It's text + print("Response:") + print(result) + else: + print("Failed to generate response") + exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/online_serving/lance/run_server.sh b/examples/online_serving/lance/run_server.sh new file mode 100755 index 000000000..bba316677 --- /dev/null +++ b/examples/online_serving/lance/run_server.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Lance online serving startup script + +MODEL="${MODEL:-bytedance-research/Lance}" +PORT="${PORT:-8091}" +DEPLOY_CONFIG="${DEPLOY_CONFIG:-vllm_omni/deploy/lance.yaml}" + +echo "Starting Lance server..." +echo "Model: $MODEL" +echo "Deploy config: $DEPLOY_CONFIG" +echo "Port: $PORT" + +vllm serve "$MODEL" --omni \ + --deploy-config "$DEPLOY_CONFIG" \ + --port "$PORT" diff --git a/recipes/ByteDance/Lance.md b/recipes/ByteDance/Lance.md new file mode 100644 index 000000000..ea5700bdc --- /dev/null +++ b/recipes/ByteDance/Lance.md @@ -0,0 +1,177 @@ +# Lance + +> Unified autoregressive + diffusion multimodal (text / image / video) + +## Summary + +- Vendor: ByteDance +- Model: [`bytedance-research/Lance`](https://huggingface.co/bytedance-research/Lance) (`Lance_3B`, `Lance_3B_Video`) +- Task: text2img, text2video, img2img (image edit), video2video (video edit), + img2text (image understanding), video2text (video understanding) +- Mode: Offline inference, Online serving (OpenAI-compatible API) +- Maintainer: Community + +## When to use this recipe + +Use this recipe to run Lance — a 3B unified AR + diffusion multimodal model +on a Qwen2.5-VL backbone — via vLLM-Omni. Lance is **BAGEL-lineage**: the +released checkpoint uses the same `*_moe_gen` Mixture-of-Transformers weight +layout as BAGEL, so vLLM-Omni reuses BAGEL's transformer core and specializes +only the ViT (Qwen2.5-VL vision), the VAE (Wan2.2) and the checkpoint layout. + +All six single-stage modalities are supported on the `Lance_3B` checkpoint; +`text2video` and `video2video` additionally need `Lance_3B_Video` for the 3-D +`latent_pos_embed` table. The two-stage AR + DiT topology is deferred and +needs `LanceConfig` / `LanceProcessor` registered in the `vllm` package (a +separate upstream PR). + +## References + +- Offline example: + [`examples/offline_inference/lance/end2end.py`](../../examples/offline_inference/lance/end2end.py) +- Online serving: + [`examples/online_serving/lance/`](../../examples/online_serving/lance/) +- Deploy config: + [`vllm_omni/deploy/lance.yaml`](../../vllm_omni/deploy/lance.yaml) +- E2E tests: + [`tests/e2e/online_serving/test_lance.py`](../../tests/e2e/online_serving/test_lance.py) +- HuggingFace model page: + [bytedance-research/Lance](https://huggingface.co/bytedance-research/Lance) +- PR: [vllm-project/vllm-omni#3710](https://github.com/vllm-project/vllm-omni/pull/3710) + (closes [#3697](https://github.com/vllm-project/vllm-omni/issues/3697)) + +## Hardware Support + +## GPU + +### 1x NVIDIA B300 / A100 80GB + +#### Environment + +- OS: Linux +- Python: 3.12 +- Driver / runtime: CUDA ≥ 12.4 +- vLLM-Omni version: 0.18.x.dev + +#### Command — text-to-image (default) + +```bash +python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance \ + --prompts "a corgi astronaut on the moon, cinematic" \ + --steps 30 --cfg-text-scale 4.0 --timestep-shift 3.5 \ + --height 1024 --width 1024 \ + --seed 42 --output ./out +``` + +Defaults match upstream `inference_lance.sh`: 30 denoising steps, +`timestep-shift 3.5`, text CFG 4.0, seed 42, 1024×1024. + +#### Command — image edit (img2img) + +```bash +python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance --modality img2img \ + --image-path /path/to/input.png \ + --prompts "Convert this into a vibrant cartoon-style illustration" \ + --steps 30 --cfg-text-scale 4.0 --timestep-shift 3.5 \ + --output ./out +``` + +The Lance-native VAE prefill scatters Wan2.2 latents into the LLM query +sequence; no separate image encoder is needed. + +#### Command — text-to-video + +```bash +python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance/Lance_3B_Video --modality text2video \ + --num-frames 25 --video-height 480 --video-width 768 \ + --prompts "a cat playing piano, cinematic" \ + --steps 30 --fps 8 --output ./out +``` + +Use the `Lance_3B_Video` subfolder for any video path so the 3-D +`latent_pos_embed` table is loaded; image / understanding paths can point at +the top-level repo and resolve the right sub-checkpoint automatically. + +#### Command — image / video understanding + +```bash +# Image → text (caption / VQA) +python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance --modality img2text \ + --image-path /path/to/photo.jpg \ + --prompts "Describe this image in detail." \ + --do-sample --text-temperature 0.8 + +# Video → text +python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance --modality video2text \ + --video-path /path/to/clip.mp4 \ + --prompts "What is happening in this video?" +``` + +Sampling is enabled by default at `--text-temperature 0.8` for the +understanding paths because Lance's greedy decoder emits an immediate EOS for +many prompts. + +#### Verification + +```bash +pytest -s -v tests/e2e/online_serving/test_lance.py +``` + +#### Notes + +- BF16 footprint: ~7 GB LLM + Qwen2.5-VL ViT + Wan2.2 VAE; comfortably fits + on a single 16 GB+ GPU for `Lance_3B`. +- `rope_scaling = {"type": "mrope", "mrope_section": [16, 24, 24]}` is wired + through `BagelRotaryEmbedding`. `text2img` uses scalar position ids + (BAGEL-equivalent); `img2text` / `video` / edit paths thread per-axis 3-D + position ids. +- `video_edit` quality is more abstract than `text2video` at the same + resolution (known position-id offset between VAE-prefill and gen-latent + blocks). Functionally correct end-to-end. + +## Online Serving + +Lance supports all single-stage modalities via the OpenAI-compatible +`/v1/chat/completions` API. + +### Launch + +```bash +bash examples/online_serving/lance/run_server.sh +# or, with overrides +MODEL=bytedance-research/Lance \ +DEPLOY_CONFIG=vllm_omni/deploy/lance.yaml \ +PORT=8091 \ + bash examples/online_serving/lance/run_server.sh +``` + +For `text2video` / `video2video`, set +`MODEL=bytedance-research/Lance/Lance_3B_Video`. + +### Send requests + +```bash +# Text-to-image +python examples/online_serving/lance/openai_chat_client.py \ + --prompt "A cute corgi astronaut on the moon, cinematic" \ + --modality text2img --output corgi.png + +# Image edit +python examples/online_serving/lance/openai_chat_client.py \ + --prompt "Convert this into a vibrant cartoon-style illustration" \ + --modality img2img --image-url path/to/photo.png \ + --output edited.png + +# Image understanding +python examples/online_serving/lance/openai_chat_client.py \ + --prompt "Describe this image" \ + --modality img2text --image-url photo.jpg +``` + +The client is shared with BAGEL — same OpenAI message format, same +`modalities` / `num_inference_steps` / `seed` / `height` / `width` knobs. diff --git a/recipes/README.md b/recipes/README.md index 005bd37ec..05b4788dd 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -29,6 +29,7 @@ recipes/ |--------|------|----------| | [`audiox/AudioX.md`](./audiox/AudioX.md) | Offline + online unified text/video→audio diffusion | 1x L4 24GB | | [`Baidu/ERNIE-Image.md`](./Baidu/ERNIE-Image.md) | Text-to-image online serving (ERNIE-Image 8B) | 1x or 2x RTX 4090 24GB | +| [`ByteDance/Lance.md`](./ByteDance/Lance.md) | Unified AR+diffusion: text/img/video gen + understanding (Lance 3B) | 1x B300 / A100 80GB | | [`fishaudio/Fish-Speech-S2-Pro.md`](./fishaudio/Fish-Speech-S2-Pro.md) | Online serving for TTS | 1x A800 80GB | | [`Helios/Helios.md`](./Helios/Helios.md) | Text-to-video, image-to-video, and video-to-video generation | 1x NVIDIA H20 | | [`inclusionAI/Ming-flash-omni-2.0.md`](./inclusionAI/Ming-flash-omni-2.0.md) | Online serving for multimodal chat + standalone TTS | 4x H100 / 1x H100 80GB | diff --git a/tests/diffusion/models/bagel/test_trajectory_recording.py b/tests/diffusion/models/bagel/test_trajectory_recording.py index 730b918e4..aa86f033a 100644 --- a/tests/diffusion/models/bagel/test_trajectory_recording.py +++ b/tests/diffusion/models/bagel/test_trajectory_recording.py @@ -19,8 +19,9 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu] NUM_TOKENS = 8 HIDDEN_DIM = 16 NUM_TIMESTEPS = 5 -# generate_image uses timesteps[:-1], so actual steps = NUM_TIMESTEPS - 1 -EXPECTED_STEPS = NUM_TIMESTEPS - 1 +# generate_image samples (num_timesteps + 1) points then drops the last, matching +# upstream lance: see Bagel.generate_image in bagel_transformer.py. +EXPECTED_STEPS = NUM_TIMESTEPS def _make_mock_bagel(mocker: MockerFixture): @@ -178,7 +179,7 @@ class TestTrajectoryRecording: bagel, args = bagel_and_args # Recompute the expected timestep schedule (mirrors generate_image logic) - ts = torch.linspace(1, 0, args["num_timesteps"]) + ts = torch.linspace(1, 0, args["num_timesteps"] + 1) shift = args.get("timestep_shift", 1.0) ts = shift * ts / (1 + (shift - 1) * ts) expected_timesteps = ts[:-1] # last element is dropped @@ -196,7 +197,7 @@ class TestTrajectoryRecording: bagel, args = bagel_and_args # Compute the number of denoising steps the same way generate_image does - ts = torch.linspace(1, 0, args["num_timesteps"]) + ts = torch.linspace(1, 0, args["num_timesteps"] + 1) shift = args.get("timestep_shift", 1.0) ts = shift * ts / (1 + (shift - 1) * ts) num_steps = len(ts) - 1 # timesteps = ts[:-1] diff --git a/tests/e2e/online_serving/test_lance.py b/tests/e2e/online_serving/test_lance.py new file mode 100644 index 000000000..9c1a40be9 --- /dev/null +++ b/tests/e2e/online_serving/test_lance.py @@ -0,0 +1,144 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""End-to-end online serving test for Lance (single-stage). + +Verifies that the Lance pipeline can serve text-to-image and image-edit +requests via the OpenAI-compatible chat completions API exposed by +``vllm-omni serve``. + +Equivalent to running: + + vllm-omni serve "bytedance-research/Lance" --omni \\ + --pipeline lance --enforce-eager --trust-remote-code --port 8091 + + # text2img + python3 examples/online_serving/lance/openai_chat_client.py \\ + --prompt "A cute corgi astronaut" --modality text2img +""" + +import base64 +import os +from io import BytesIO + +import pytest +from vllm.assets.image import ImageAsset + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServerParams +from vllm_omni.diffusion.models.lance.prompts import ( + VIDEO_PAD, + VISION_END, + VISION_START, + render_lance_prompt, +) + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +MODEL = "bytedance-research/Lance" + +TEXT2IMG_PROMPT = "A cute corgi astronaut on the moon, cinematic" +IMG2IMG_PROMPT = "Convert this into a vibrant cartoon-style illustration" + +# Lance vision block: <|vision_start|><|video_pad|><|vision_end|> +_VISION_BLOCK = f"{VISION_START}{VIDEO_PAD}{VISION_END}" + +# Lance is single-stage diffusion — no deploy YAML. Pass the engine +# knobs that used to live in ``vllm_omni/deploy/lance.yaml`` via CLI +# flags so ``create_default_diffusion`` builds the stage config. +_LANCE_SERVE_ARGS = [ + "--pipeline", + "lance", + "--max-num-batched-tokens", + "32768", + "--max-num-seqs", + "1", + "--enforce-eager", + "--trust-remote-code", + "--no-enable-prefix-caching", + "--no-async-chunk", +] + +test_params = [ + OmniServerParams( + model=MODEL, + server_args=_LANCE_SERVE_ARGS, + stage_init_timeout=300, + ), +] + + +def _build_text2img_messages(prompt: str) -> list[dict]: + """Build OpenAI-format messages for text2img generation.""" + rendered = render_lance_prompt("t2i", prompt) + return [ + { + "role": "user", + "content": [{"type": "text", "text": rendered}], + } + ] + + +def _build_img2img_messages(prompt: str, image_b64: str) -> list[dict]: + """Build OpenAI-format messages for img2img generation.""" + rendered = render_lance_prompt("image_edit", prompt, vision_token=_VISION_BLOCK) + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": rendered}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}, + }, + ], + } + ] + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_lance_text2img_online(omni_server, openai_client) -> None: + """Lance text2img via the OpenAI-compatible chat completions API.""" + request_config = { + "model": omni_server.model, + "messages": _build_text2img_messages(TEXT2IMG_PROMPT), + "modalities": ["image"], + "extra_body": { + "height": 512, + "width": 512, + "num_inference_steps": 2, + "guidance_scale": 0.0, + "seed": 42, + }, + } + + openai_client.send_diffusion_request(request_config) + + +@pytest.mark.core_model +@pytest.mark.advanced_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_lance_img2img_online(omni_server, openai_client) -> None: + """Lance image_edit via the OpenAI-compatible chat completions API.""" + input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB") + buffer = BytesIO() + input_image.save(buffer, format="JPEG") + image_b64 = base64.b64encode(buffer.getvalue()).decode("utf-8") + + request_config = { + "model": omni_server.model, + "messages": _build_img2img_messages(IMG2IMG_PROMPT, image_b64), + "modalities": ["image"], + "extra_body": { + "num_inference_steps": 2, + "guidance_scale": 0.0, + "seed": 42, + }, + } + + openai_client.send_diffusion_request(request_config) diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index 140bddf45..938dd74e7 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -65,6 +65,11 @@ _OMNI_PIPELINES: dict[str, tuple[str, str]] = { "vllm_omni.model_executor.models.bagel.pipeline", "BAGEL_SINGLE_STAGE_PIPELINE", ), + # Lance (ByteDance) — BAGEL-lineage unified AR+diffusion, single-stage DiT. + "lance": ( + "vllm_omni.model_executor.models.lance.pipeline", + "LANCE_PIPELINE", + ), "glm_image": ( "vllm_omni.model_executor.models.glm_image.pipeline", "GLM_IMAGE_PIPELINE", diff --git a/vllm_omni/deploy/lance.yaml b/vllm_omni/deploy/lance.yaml new file mode 100644 index 000000000..25e0afd4d --- /dev/null +++ b/vllm_omni/deploy/lance.yaml @@ -0,0 +1,20 @@ +# Lance (ByteDance) — single-stage diffusion pipeline selector. +# +# Lance's HF config.json is descriptive metadata (no ``model_type``), +# so the loader can't auto-detect from the model dir alone. This YAML +# only tells the loader "use the Lance pipeline" — one stage, no multi- +# stage orchestration. + +pipeline: lance +async_chunk: false + +stages: + - stage_id: 0 + max_num_batched_tokens: 32768 + max_num_seqs: 1 + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: false + devices: "0" + default_sampling_params: + seed: 42 diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 5fe2d4061..8e9bdbf32 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -847,6 +847,21 @@ class OmniDiffusionConfig: self.supports_multimodal_inputs = metadata.supports_multimodal_inputs self.max_multimodal_image_inputs = metadata.max_multimodal_image_inputs + @staticmethod + def _looks_like_lance_subfolder(model: str | None) -> bool: + """Return True when ``--model`` points at a Lance per-component subfolder. + + Lance's HF repo bundles ``Lance_3B/``, ``Lance_3B_Video/`` and + ``Qwen2.5-VL-ViT/`` under a single top-level ``config.json``; users may + reasonably hand the AR-style sub-checkpoint path directly. The + ``LancePipeline`` constructor knows to walk up to the repo root from + either subfolder name. + """ + if not model: + return False + base = os.path.basename(str(model).rstrip("/")) + return base in {"Lance_3B", "Lance_3B_Video"} + def enrich_config(self) -> None: """Load model metadata from HuggingFace and populate config fields. @@ -900,6 +915,15 @@ class OmniDiffusionConfig: else: cfg = get_hf_file_to_dict("config.json", self.model) if cfg is None: + # Lance ships its top-level config.json one directory above + # the per-checkpoint subfolders (``Lance_3B/`` or + # ``Lance_3B_Video/``). Try to recover that case before + # raising. + if self._looks_like_lance_subfolder(self.model): + self.model_class_name = "LancePipeline" + self.set_tf_model_config(TransformerConfig()) + self.update_multimodal_support() + return raise ValueError(f"Could not find config.json or model_index.json for model {self.model}") self.set_tf_model_config(TransformerConfig.from_dict(cfg)) @@ -910,6 +934,19 @@ class OmniDiffusionConfig: self.model_class_name = "BagelPipeline" self.set_tf_model_config(TransformerConfig()) self.update_multimodal_support() + elif ( + model_type == "lance" + or "LancePipeline" in architectures + or cfg.get("model_name") == "Lance" + or self._looks_like_lance_subfolder(self.model) + ): + # Lance ships a non-HF top-level config.json (model_name only) + # plus per-component subfolders; resolve to the Lance pipeline. + # Also accept --model pointing directly at the ``Lance_3B`` or + # ``Lance_3B_Video`` subfolder by walking up to the repo root. + self.model_class_name = "LancePipeline" + self.set_tf_model_config(TransformerConfig()) + self.update_multimodal_support() elif model_type == "neo_chat": self.model_class_name = "SenseNovaU1Pipeline" self.tf_model_config = TransformerConfig() diff --git a/vllm_omni/diffusion/models/bagel/bagel_transformer.py b/vllm_omni/diffusion/models/bagel/bagel_transformer.py index c61db037b..4039e06ac 100644 --- a/vllm_omni/diffusion/models/bagel/bagel_transformer.py +++ b/vllm_omni/diffusion/models/bagel/bagel_transformer.py @@ -106,7 +106,16 @@ class BagelRotaryEmbedding(nn.Module): linear, dynamic-NTK, YaRN, …), we delegate the ``inv_freq`` / ``attention_scaling`` computation to HF's ``ROPE_INIT_FUNCTIONS`` so that the frequency basis and scaling factor are identical to the - original checkpoint. This module has no learnable parameters. + original checkpoint. + + For Qwen2.5-VL-style multimodal RoPE (``rope_scaling.rope_type == "mrope"``) + the ``inv_freq`` basis is the standard default-rope one; the difference + is that position ids are 3-D ``(t, h, w)`` per token and the + ``mrope_section`` describes how the head dimension is split across axes. + This module accepts either 2-D scalar position ids ``(B, S)`` or 3-D + multimodal position ids ``(B, 3, S)`` and dispatches accordingly so the + same module works for both BAGEL (1-D rope) and Lance (Qwen2.5-VL mrope). + This module has no learnable parameters. """ def __init__(self, config): @@ -120,10 +129,17 @@ class BagelRotaryEmbedding(nn.Module): rope_type = ( rope_scaling.get("rope_type") or rope_scaling.get("type") or rope_parameters.get("rope_type") or "default" ) + # Cache mrope_section for the forward-time section-split. + self._mrope_section: list[int] | None = None + if rope_type == "mrope": + section = rope_scaling.get("mrope_section") or rope_parameters.get("mrope_section") + if section is None: + raise ValueError("rope_scaling.rope_type == 'mrope' requires 'mrope_section'.") + self._mrope_section = list(section) - if rope_type == "default": - # transformers>=5.0 removed the 'default' entry from - # ROPE_INIT_FUNCTIONS; use the plain sinusoidal formula. + if rope_type in ("default", "mrope"): + # mrope shares the default sinusoidal frequency basis; the + # multimodal split happens at forward time, not at init. rope_theta = ( getattr(config, "rope_theta", None) or rope_parameters.get("rope_theta") @@ -147,11 +163,36 @@ class BagelRotaryEmbedding(nn.Module): Args: x: Input tensor (only used for dtype inference). - position_ids: Position indices, shape (batch_size, seq_len). + position_ids: Either 2-D scalar ``(batch_size, seq_len)`` for plain + 1-D RoPE, or 3-D multimodal ``(batch_size, 3, seq_len)`` for + Qwen2.5-VL-style mRoPE. The latter is auto-detected from + ``position_ids.ndim``. Returns: cos, sin: Rotary embeddings, each of shape (batch_size, seq_len, dim). """ + if position_ids.ndim == 3 and self._mrope_section is not None: + # multimodal path: position_ids is (B, 3, S) with rows = (t, h, w). + # Compute per-axis frequencies, then assemble the per-section + # rotary basis matching Qwen2-VL's ``apply_multimodal_rotary_pos_emb``. + B = position_ids.shape[0] + inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(B, 3, -1, 1) + position_ids_expanded = position_ids[:, :, None, :].float() + freqs = (inv_freq_expanded @ position_ids_expanded).transpose(2, 3) + # ``freqs`` is (B, 3, S, head_dim/2); double along the last axis + # to get the full ``head_dim`` rotary basis. + emb = torch.cat((freqs, freqs), dim=-1) # (B, 3, S, head_dim) + cos_per_axis = emb.cos() * self.attention_scaling + sin_per_axis = emb.sin() * self.attention_scaling + # ``mrope_section`` (e.g. [16, 24, 24] for Qwen2.5-VL) sums to + # head_dim/2; doubled it sums to head_dim and cycles axis = i % 3. + sec_full = self._mrope_section * 2 + cos_split = cos_per_axis.split(sec_full, dim=-1) + sin_split = sin_per_axis.split(sec_full, dim=-1) + cos = torch.cat([c[:, i % 3] for i, c in enumerate(cos_split)], dim=-1) + sin = torch.cat([s[:, i % 3] for i, s in enumerate(sin_split)], dim=-1) + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) position_ids_expanded = position_ids[:, None, :].float() freqs = (inv_freq_expanded @ position_ids_expanded).transpose(1, 2) @@ -541,8 +582,12 @@ class PackedAttentionMoT(nn.Module): q = q.view(-1, self.num_heads, self.head_dim) k = k.view(-1, self.num_kv_heads, self.head_dim) v = v.view(-1, self.num_kv_heads, self.head_dim) - q = self.q_norm(q) - k = self.k_norm(k) + # Pre-merge code cast to float32 before q_norm/k_norm — bf16 + # RMSNorm accumulation drift compounds across segmented prefill + # (x2t builds the cache via 3+ sequential forward_cache_update_* + # calls), producing degenerate token repetition in generate_text. + q = self.q_norm(q.to(torch.float32)) + k = self.k_norm(k.to(torch.float32)) cos, sin = [x[..., : self.head_dim // 2] for x in packed_query_position_embeddings] q = self.rotary_op(q.to(cos.dtype).unsqueeze(0), cos, sin).squeeze(0) @@ -557,16 +602,52 @@ class PackedAttentionMoT(nn.Module): cache_v = past_key_values.value_cache[self.layer_idx] full_k = torch.cat([cache_k, k], dim=0) full_v = torch.cat([cache_v, v], dim=0) + cache_len = cache_k.shape[0] else: full_k = k full_v = v + cache_len = 0 - attn = self.attn_causal if is_causal else self.attn_noncausal - attn_out = attn( - q.unsqueeze(0), - full_k.unsqueeze(0), - full_v.unsqueeze(0), - ) + if is_causal and cache_len > 0: + # PyTorch SDPA's ``is_causal=True`` with ``Q != K`` uses + # ``tril(diagonal=0)`` — top-left aligned, so for Q=1 against a + # long cache only ``q_0 -> k_0`` is unmasked (degenerate + # generate_text output). Build the correct bottom-right + # aligned mask manually: ``mask[i, j] = -inf if j > cache_len + # + i`` so each new query attends to all of ``cache + self + # up to i``. Bypass ``DiffusionAttention`` entirely — its + # ``_maybe_reshape_attn_mask`` helper only handles 2-D + # ``(B, K)`` shape, not ``(Q, K)``, so the reshape path + # silently drops the per-query mask for Q>1. + Q_len = q.shape[0] + K_len = full_k.shape[0] + arange_q = torch.arange(Q_len, device=q.device).unsqueeze(1) + arange_k = torch.arange(K_len, device=q.device).unsqueeze(0) + mask = arange_k > (cache_len + arange_q) # (Q, K) bool + attn_bias = torch.zeros(Q_len, K_len, dtype=q.dtype, device=q.device) + attn_bias.masked_fill_(mask, float("-inf")) + # Permute to (B=1, H, S, D) for SDPA. + q_4d = q.unsqueeze(0).permute(0, 2, 1, 3) + k_4d = full_k.unsqueeze(0).permute(0, 2, 1, 3) + v_4d = full_v.unsqueeze(0).permute(0, 2, 1, 3) + attn_out_4d = torch.nn.functional.scaled_dot_product_attention( + q_4d, + k_4d, + v_4d, + attn_mask=attn_bias.unsqueeze(0).unsqueeze(0), # (1, 1, Q, K) + dropout_p=0.0, + is_causal=False, + scale=1.0 / (self.head_dim**0.5), + enable_gqa=(self.num_heads != self.num_kv_heads), + ) + attn_out = attn_out_4d.permute(0, 2, 1, 3) + else: + attn = self.attn_causal if is_causal else self.attn_noncausal + attn_out = attn( + q.unsqueeze(0), + full_k.unsqueeze(0), + full_v.unsqueeze(0), + ) attn_out = attn_out.squeeze(0).reshape(-1, self.q_size) attn_out, _ = self.o_proj(attn_out) @@ -1503,23 +1584,6 @@ class Bagel(nn.Module): return generation_input - @staticmethod - def _merge_naive_caches(caches: list) -> NaiveCache: - """Merge multiple NaiveCache objects by concatenating KV tensors per layer.""" - if not caches: - # Handle empty list case gracefully if desired, - # though original code also crashed on this. - return NaiveCache(0) - - num_layers = len(caches[0].key_cache) - merged = NaiveCache(num_layers) - for layer_idx in range(num_layers): - key_parts = [c.key_cache[layer_idx] for c in caches if c.key_cache[layer_idx] is not None] - val_parts = [c.value_cache[layer_idx] for c in caches if c.value_cache[layer_idx] is not None] - merged.key_cache[layer_idx] = torch.cat(key_parts, dim=0) if key_parts else None - merged.value_cache[layer_idx] = torch.cat(val_parts, dim=0) if val_parts else None - return merged - def prepare_start_tokens(self, curr_kvlens, curr_rope, new_token_ids): """Prepare start tokens for autoregressive text generation. @@ -1614,10 +1678,26 @@ class Bagel(nn.Module): return_trajectory_latents: bool = False, scheduler: object | None = None, scheduler_kwargs: dict | None = None, + # Lance i2v: tokens to freeze at their initial (encoded-image) + # value throughout the denoise loop. Matches upstream's + # ``mse_loss_indexes``-exclusion behaviour from PR #33: cond + # positions are never updated and get ``timestep=0``. + frame_condition_token_indexes: torch.LongTensor | None = None, ): x_t = packed_init_noises + # Snapshot the pinned subtensor BEFORE the denoise loop touches + # x_t; the cond positions in packed_init_noises hold the + # VAE-encoded conditioning latent that must be preserved verbatim. + pinned_x_t = None + if frame_condition_token_indexes is not None: + frame_condition_token_indexes = frame_condition_token_indexes.to(x_t.device).long() + pinned_x_t = x_t[frame_condition_token_indexes].clone() - timesteps = torch.linspace(1, 0, num_timesteps, device=x_t.device) + # Use num_timesteps + 1 sample points so we get `num_timesteps` denoise + # steps after dropping the terminal t=0 (which has no dt). Upstream + # Lance / BAGEL both use this convention; without the +1 we silently + # run one fewer denoise iteration than the user asked for. + timesteps = torch.linspace(1, 0, num_timesteps + 1, device=x_t.device) timesteps = timestep_shift * timesteps / (1 + (timestep_shift - 1) * timesteps) dts = timesteps[:-1] - timesteps[1:] timesteps = timesteps[:-1] @@ -1669,6 +1749,12 @@ class Bagel(nn.Module): trajectory_latents.append(x_t.clone()) for i, t in enumerate(timesteps): timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device) + if frame_condition_token_indexes is not None: + # Cond positions stay at t=0 (clean signal). Matches upstream + # PR #33 lance.py line 1605: + # timestep[current_vae_mse_indexes_local_in_vae] = t + # (cond positions remain at the ``torch.zeros`` init value). + timestep[frame_condition_token_indexes] = 0.0 in_cfg_window = t > cfg_interval[0] and t <= cfg_interval[1] cfg_text_scale_ = cfg_text_scale if in_cfg_window else 1.0 cfg_img_scale_ = cfg_img_scale if in_cfg_window else 1.0 @@ -1732,6 +1818,12 @@ class Bagel(nn.Module): trajectory_latents.append(x_t.clone()) for i, t in enumerate(timesteps): timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device) + if frame_condition_token_indexes is not None: + # Cond positions stay at t=0 (clean signal). Matches upstream + # PR #33 lance.py line 1605: + # timestep[current_vae_mse_indexes_local_in_vae] = t + # (cond positions remain at the ``torch.zeros`` init value). + timestep[frame_condition_token_indexes] = 0.0 v_t = self.forward_single_branch( x_t=x_t, timestep=timestep, @@ -1758,46 +1850,31 @@ class Bagel(nn.Module): unpacked_latent = x_t.split((packed_seqlens - 2).tolist()) return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs - # ── Batched CFG mode (cfg_parallel_size=1, no SP) ── - cfg_batched = None + # ── Sequential CFG mode (cfg_parallel_size=1, no SP) ── + # Each CFG branch runs its own LLM forward; we just need the + # per-branch packed_position_ids and past_key_values for + # ``Bagel.forward`` to dispatch through. + cfg_branches: dict | None = None if use_cfg_text: - seq_len = int(packed_seqlens.sum()) - - # Branch 0: main (gen_context), always present - branches_pid = [packed_position_ids] - branches_cache = [past_key_values] - - # Branch 1: cfg_text (unconditional text), always present when use_cfg_text - branches_pid.append(cfg_text_packed_position_ids) - branches_cache.append(cfg_text_past_key_values) - - # Branch 2: cfg_img (text-only, no image), optional + branches_pid = [packed_position_ids, cfg_text_packed_position_ids] + branches_cache = [past_key_values, cfg_text_past_key_values] if use_cfg_img: branches_pid.append(cfg_img_packed_position_ids) branches_cache.append(cfg_img_past_key_values) - - num_branches = len(branches_cache) - - cfg_batched = { - "num_branches": num_branches, - "seq_len": seq_len, - "batched_query_lens": packed_seqlens.repeat(num_branches), - "batched_position_ids": torch.cat(branches_pid), - "batched_text_indexes": torch.cat( - [packed_text_indexes + b_idx * seq_len for b_idx in range(num_branches)] - ), - "batched_vae_indexes": torch.cat( - [packed_vae_token_indexes + b_idx * seq_len for b_idx in range(num_branches)] - ), - "merged_cache": self._merge_naive_caches(branches_cache), - } + cfg_branches = {"pids": branches_pid, "caches": branches_cache} if return_trajectory_latents and len(timesteps) > 0: trajectory_latents.append(x_t.clone()) for i, t in enumerate(timesteps): timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device) + if frame_condition_token_indexes is not None: + # Cond positions stay at t=0 (clean signal). Matches upstream + # PR #33 lance.py line 1605: + # timestep[current_vae_mse_indexes_local_in_vae] = t + # (cond positions remain at the ``torch.zeros`` init value). + timestep[frame_condition_token_indexes] = 0.0 if t > cfg_interval[0] and t <= cfg_interval[1]: cfg_text_scale_ = cfg_text_scale cfg_img_scale_ = cfg_img_scale @@ -1818,7 +1895,7 @@ class Bagel(nn.Module): cfg_renorm_type=cfg_renorm_type, cfg_text_scale=cfg_text_scale_, cfg_img_scale=cfg_img_scale_, - cfg_batched=cfg_batched, + cfg_branches=cfg_branches, ) if scheduler is not None: @@ -1828,6 +1905,12 @@ class Bagel(nn.Module): trajectory_log_probs.append(out.log_prob) else: x_t = x_t - v_t.to(x_t.device) * dts[i] # velocity pointing from data to noise + if pinned_x_t is not None: + # i2v: restore cond positions to their encoded-image + # latent. Matches upstream PR #33 lance.py line 1712: + # x_t[mse_indexes] = x_t[mse_indexes] - v_t[mse_indexes] * dts[i] + # (cond positions are excluded from the update). + x_t[frame_condition_token_indexes] = pinned_x_t if return_trajectory_latents: trajectory_latents.append(x_t.clone()) trajectory_timesteps.append(timesteps[i]) @@ -1859,6 +1942,7 @@ class Bagel(nn.Module): return_trajectory_latents: bool = False, scheduler: object | None = None, scheduler_kwargs: dict | None = None, + frame_condition_token_indexes: torch.LongTensor | None = None, ): """CFG parallel denoising loop: each rank computes one CFG branch. @@ -1918,6 +2002,12 @@ class Bagel(nn.Module): for i, t in enumerate(timesteps): timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device) + if frame_condition_token_indexes is not None: + # Cond positions stay at t=0 (clean signal). Matches upstream + # PR #33 lance.py line 1605: + # timestep[current_vae_mse_indexes_local_in_vae] = t + # (cond positions remain at the ``torch.zeros`` init value). + timestep[frame_condition_token_indexes] = 0.0 use_cfg_this_step = t > cfg_interval[0] and t <= cfg_interval[1] and cfg_text_scale > 1.0 if use_cfg_this_step: @@ -2070,7 +2160,7 @@ class Bagel(nn.Module): packed_sequence = packed_text_embedding.new_zeros((int(local_seqlens.sum()), self.hidden_size)) packed_sequence[local_text_indexes] = packed_text_embedding - assert timestep.unique().shape[0] == 1 + # i2v relaxes this: per-token timestep (cond=0, noncond=t) is valid. packed_pos_embed = self.latent_pos_embed(local_vae_pos_ids) local_timestep = timestep[: local_x_t.shape[0]] packed_timestep_embeds = self.time_embedder(local_timestep) @@ -2107,7 +2197,7 @@ class Bagel(nn.Module): packed_sequence = packed_text_embedding.new_zeros((sum(packed_seqlens), self.hidden_size)) packed_sequence[packed_text_indexes] = packed_text_embedding - assert timestep.unique().shape[0] == 1 + # i2v relaxes this: per-token timestep (cond=0, noncond=t) is valid. packed_pos_embed = self.latent_pos_embed(packed_vae_position_ids) packed_timestep_embeds = self.time_embedder(timestep) x_t_emb = self.vae2llm(x_t) + packed_timestep_embeds + packed_pos_embed @@ -2149,7 +2239,7 @@ class Bagel(nn.Module): cfg_renorm_type: str = "global", cfg_text_scale: float = 1.0, cfg_img_scale: float = 1.0, - cfg_batched: dict | None = None, + cfg_branches: dict | None = None, ): # Build query sequence (identical for all CFG branches) packed_text_embedding = self.language_model.forward( @@ -2159,7 +2249,7 @@ class Bagel(nn.Module): packed_sequence = packed_text_embedding.new_zeros((sum(packed_seqlens), self.hidden_size)) packed_sequence[packed_text_indexes] = packed_text_embedding - assert timestep.unique().shape[0] == 1 + # i2v relaxes this: per-token timestep (cond=0, noncond=t) is valid. packed_pos_embed = self.latent_pos_embed(packed_vae_position_ids) packed_timestep_embeds = self.time_embedder(timestep) x_t = self.vae2llm(x_t) + packed_timestep_embeds + packed_pos_embed @@ -2170,55 +2260,40 @@ class Bagel(nn.Module): extra_inputs = {} if self.use_moe: extra_inputs["mode"] = "gen" + extra_inputs["packed_vae_token_indexes"] = packed_vae_token_indexes + extra_inputs["packed_text_indexes"] = packed_text_indexes use_cfg = cfg_text_scale > 1.0 cfg_text_v_t = None cfg_img_v_t = None - if use_cfg and cfg_batched is not None: - # ── Batched CFG: single LLM forward for all branches ── - seq_len = cfg_batched["seq_len"] - num_branches = cfg_batched["num_branches"] + if use_cfg and cfg_branches is not None: + # Sequential per-branch CFG forwards (matches upstream lance.py). + # The previous batched path concatenated cond + cfg into one LLM + # forward, but the block-diagonal attention mask was lost when + # PR #3728 dropped flash_attn_varlen, so branches leaked into + # each other. Running each branch through its own forward is + # numerically identical to upstream. + def _run_branch(branch_pkv, branch_pids): + out = self.language_model.forward( + packed_query_sequence=packed_sequence, + query_lens=packed_seqlens, + packed_query_position_ids=branch_pids, + past_key_values=branch_pkv, + update_past_key_values=False, + is_causal=False, + **extra_inputs, + ) + return self.llm2vae(out.packed_query_sequence)[packed_vae_token_indexes] - batched_sequence = packed_sequence.repeat(num_branches, 1) - - if self.use_moe: - extra_inputs["packed_text_indexes"] = cfg_batched["batched_text_indexes"] - extra_inputs["packed_vae_token_indexes"] = cfg_batched["batched_vae_indexes"] - - output = self.language_model.forward( - packed_query_sequence=batched_sequence, - query_lens=cfg_batched["batched_query_lens"], - packed_query_position_ids=cfg_batched["batched_position_ids"], - past_key_values=cfg_batched["merged_cache"], - update_past_key_values=False, - is_causal=False, - **extra_inputs, - ) - - # Extract per-branch velocities from batched output - all_hidden = output.packed_query_sequence - assert all_hidden.shape[0] == seq_len * num_branches, ( - f"Expected packed sequence length {seq_len * num_branches}, but got {all_hidden.shape[0]}" - ) - - v_t = self.llm2vae(all_hidden[:seq_len])[packed_vae_token_indexes] - - branch_idx = 1 - cfg_text_v_t = self.llm2vae(all_hidden[branch_idx * seq_len : (branch_idx + 1) * seq_len])[ - packed_vae_token_indexes - ] - branch_idx += 1 - if cfg_img_scale > 1.0: - cfg_img_v_t = self.llm2vae(all_hidden[branch_idx * seq_len : (branch_idx + 1) * seq_len])[ - packed_vae_token_indexes - ] + branches_pids = cfg_branches["pids"] + branches_caches = cfg_branches["caches"] + v_t = _run_branch(branches_caches[0], branches_pids[0]) + cfg_text_v_t = _run_branch(branches_caches[1], branches_pids[1]) + if cfg_img_scale > 1.0 and len(branches_caches) > 2: + cfg_img_v_t = _run_branch(branches_caches[2], branches_pids[2]) else: - # ── Single forward (no CFG or outside cfg_interval) ── - if self.use_moe: - extra_inputs["packed_vae_token_indexes"] = packed_vae_token_indexes - extra_inputs["packed_text_indexes"] = packed_text_indexes - + # Single forward (no CFG or outside cfg_interval). output = self.language_model.forward( packed_query_sequence=packed_sequence, query_lens=packed_seqlens, @@ -2228,8 +2303,7 @@ class Bagel(nn.Module): is_causal=False, **extra_inputs, ) - v_t = self.llm2vae(output.packed_query_sequence) - v_t = v_t[packed_vae_token_indexes] + v_t = self.llm2vae(output.packed_query_sequence)[packed_vae_token_indexes] # ── CFG combination ── if use_cfg: diff --git a/vllm_omni/diffusion/models/bagel/pipeline_bagel.py b/vllm_omni/diffusion/models/bagel/pipeline_bagel.py index a62d2a75a..26451615e 100644 --- a/vllm_omni/diffusion/models/bagel/pipeline_bagel.py +++ b/vllm_omni/diffusion/models/bagel/pipeline_bagel.py @@ -319,6 +319,27 @@ class BagelPipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipelineProf image = (image * 0.5 + 0.5).clamp(0, 1)[0].permute(1, 2, 0) * 255 return Image.fromarray(image.to(torch.uint8).cpu().numpy()) + def _regen_init_noise_on_device(self, gen_input: dict, seed: int | None) -> None: + """Resample ``gen_input["packed_init_noises"]`` on-device with a fresh + per-call ``torch.Generator``. + + ``Bagel.prepare_input`` (and the Lance video equivalent) call + ``torch.randn`` with no device or generator, falling back to CPU+fp32 + via the global RNG. Upstream Lance samples directly on CUDA+bf16 via + ``torch.Generator(device=cuda).manual_seed(seed)`` (lance.py:1536), + so for the same seed the two sides land on different noise streams. + Mutates ``gen_input`` in place; no-op if seed is unset or device is CPU. + """ + if seed is None or self.device.type != "cuda": + return + ref = gen_input["packed_init_noises"] + gen_input["packed_init_noises"] = torch.randn( + ref.shape, + generator=torch.Generator(device=self.device).manual_seed(int(seed)), + device=self.device, + dtype=self.od_config.dtype, + ) + @torch.inference_mode() def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput: if len(req.prompts) > 1: @@ -355,7 +376,7 @@ class BagelPipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipelineProf gen_params = BagelGenParams( num_timesteps=int(req.sampling_params.num_inference_steps or 50), - timestep_shift=3.0, + timestep_shift=float(extra_args.get("timestep_shift", 3.0)), cfg_text_scale=cfg_text_scale, cfg_img_scale=cfg_img_scale, cfg_interval=cfg_interval, @@ -756,6 +777,8 @@ class BagelPipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipelineProf if torch.is_tensor(v): generation_input[k] = v.to(self.device) + self._regen_init_noise_on_device(generation_input, req.sampling_params.seed) + # text cfg generation_input_cfg_text = self.bagel.prepare_vae_latent_cfg( curr_kvlens=cfg_text_context["kv_lens"], @@ -820,6 +843,11 @@ class BagelPipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipelineProf custom = {} if think_text is not None: custom["think_text"] = think_text + # Mirror the PIL image into ``custom_output`` so callers reading via + # the orchestrator IPC boundary (which strips the bare ``output`` + # field) can still recover the result. ``video_frames`` already + # uses this pattern. + custom["image"] = img return DiffusionOutput( output=img, diff --git a/vllm_omni/diffusion/models/lance/__init__.py b/vllm_omni/diffusion/models/lance/__init__.py new file mode 100644 index 000000000..ca5e1ce66 --- /dev/null +++ b/vllm_omni/diffusion/models/lance/__init__.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Lance (ByteDance) diffusion model components. + +Lance is a unified autoregressive + diffusion multimodal model on a +Qwen2.5-VL-3B backbone. Architecturally it is the BAGEL family (ByteDance +Mixture-of-Transformers): the released ``Lance_3B`` checkpoint uses the exact +same ``*_moe_gen`` MoT weight layout as BAGEL, plus ``vae2llm`` / ``llm2vae`` / +``time_embedder`` / ``latent_pos_embed`` connectors. The deltas vs BAGEL are: + + * backbone is Qwen2.5-VL (mRoPE) instead of Qwen2, + * understanding ViT is Qwen2.5-VL vision (not SigLIP), loaded from the base + ``Qwen/Qwen2.5-VL-3B-Instruct`` rather than from the Lance checkpoint, + * VAE is Wan2.2 (reused from the vLLM-Omni WAN path) instead of the BAGEL AE, + * video path adds 3D latent position embeddings (follow-up; this module + implements the image path first). + +Because Lance is BAGEL-lineage, the transformer core is reused verbatim from +``vllm_omni.diffusion.models.bagel.bagel_transformer`` and only the pipeline +wiring (ViT / VAE / checkpoint layout) is specialized here. +""" + +from vllm_omni.diffusion.models.lance.pipeline_lance import ( + LancePipeline, + get_lance_post_process_func, +) + +__all__ = [ + "LancePipeline", + "get_lance_post_process_func", +] diff --git a/vllm_omni/diffusion/models/lance/lance_transformer.py b/vllm_omni/diffusion/models/lance/lance_transformer.py new file mode 100644 index 000000000..012ba3f42 --- /dev/null +++ b/vllm_omni/diffusion/models/lance/lance_transformer.py @@ -0,0 +1,1550 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Lance transformer pieces. + +The Lance LLM is BAGEL's Qwen2-MoT transformer verbatim — the released +``Lance_3B`` checkpoint uses the identical ``*_moe_gen`` / ``q_norm`` / +``vae2llm`` / ``llm2vae`` / ``time_embedder`` / ``latent_pos_embed`` layout, so +``Bagel`` / ``Qwen2MoTForCausalLM`` / ``Qwen2MoTConfig`` / ``NaiveCache`` are +re-exported unchanged. Only the understanding ViT differs (Qwen2.5-VL vision +instead of SigLIP) and the video path adds a 3D latent position embedding. + +This module also provides :class:`LanceBagel`, a thin :class:`Bagel` subclass +that overrides the two ViT entry points to consume Qwen2.5-VL's packed +``pixel_values`` + ``image_grid_thw`` layout directly (BAGEL itself assumes +SigLIP-style ``(C, H, W)`` tensors that get patchified inside the model). + +NOTE ON mRoPE: Lance's backbone is Qwen2.5-VL and its understanding / +video paths use multimodal RoPE (``mrope_section=[16,24,24]``). BAGEL's +``BagelRotaryEmbedding`` is plain 1-D RoPE on scalar position ids. For the +text2img generation path Lance assigns scalar positions (the gen latent block +shares a single rope position, same as BAGEL), so the reused rotary is +correct there. Full mRoPE for the x2t / video understanding path is a +follow-up — see ``LancePositionEmbedding3D``. +""" + +from __future__ import annotations + +import numpy as np +import torch +from torch import nn + +# Lance LLM == BAGEL Qwen2-MoT: reuse verbatim (checkpoint weight names match). +from vllm_omni.diffusion.models.bagel.bagel_transformer import ( # noqa: F401 + Bagel, + BaseNavitOutputWithPast, + MLPconnector, + NaiveCache, + PackedAttentionMoT, + PositionEmbedding, + Qwen2MoTConfig, + Qwen2MoTDecoderLayer, + Qwen2MoTForCausalLM, + Qwen2MoTModel, + TimestepEmbedder, + get_1d_sincos_pos_embed_from_grid, + patchify, +) + +__all__ = [ + "Bagel", + "BaseNavitOutputWithPast", + "LanceBagel", + "LanceIdentityConnector", + "LancePositionEmbedding3D", + "LanceQwen2_5_VLNaViTWrapper", + "LanceZeroVitPosEmbed", + "MLPconnector", + "NaiveCache", + "PackedAttentionMoT", + "PositionEmbedding", + "Qwen2MoTConfig", + "Qwen2MoTDecoderLayer", + "Qwen2MoTForCausalLM", + "Qwen2MoTModel", + "TimestepEmbedder", + "patchify", +] + +# mRoPE temporal scaling constants for Lance (Qwen2.5-VL backbone). +# Upstream Lance computes vision rope positions as +# ``t_index = frame_idx * tokens_per_second * second_per_grid_t`` so adjacent +# latent frames sit at well-separated rope coordinates. Lance bundles a +# ``Qwen2.5-VL-ViT/config.json`` with ``tokens_per_second: 2`` and uses +# ``second_per_grid_ts = 1.0`` in its inference script. +LANCE_TOKENS_PER_SECOND = 2 +LANCE_SECONDS_PER_GRID = 1.0 + +# Upstream Lance ViT preprocessing (see lance-upstream +# ``data/datasets_custom/validation_dataset.py`` + ``data/video/transforms/``): +# ``VideoTransform(resolution=resolution_vit, mode='bucket', +# divisible_crop_size=28, stride_spatial=16, aspect_ratios=[...])``. +# For ``image_768res`` (Lance's default for 768px image gen), resolution_vit=672. +# Resizes to the nearest aspect-ratio bucket (max_area=672², stride=16), then +# center-crops to dims divisible by 28 (=patch_size * spatial_merge_size). +# Standard Qwen2VLImageProcessor's smart-resize is NOT used by upstream — the +# image enters the ViT at the bucket-cropped dimensions. +LANCE_VIT_BUCKET_RESOLUTION = 672 +LANCE_VIT_BUCKET_STRIDE = 16 +LANCE_VIT_DIVISIBLE_CROP = 28 # patch_size (14) * spatial_merge_size (2) +LANCE_VIT_ASPECT_RATIOS = ("21:9", "16:9", "4:3", "1:1", "3:4", "9:16") + +# Video-specific bucket parameters (resolution=video_480p — Lance's default for +# t2v / video_edit at 12 fps). See lance-upstream +# ``data/datasets_custom/validation_dataset.py`` lines 128-156: +# VAE side: resolution_vae=640, divisible_crop_size=16, mean=0.5, std=0.5 +# ViT side: resolution_vit=616, divisible_crop_size=28, +# mean=Qwen2.5-VL CLIP-norm, std=Qwen2.5-VL CLIP-norm +LANCE_VIDEO_VAE_RESOLUTION = 640 +LANCE_VIDEO_VAE_DIVISIBLE_CROP = 16 +LANCE_VIDEO_VIT_RESOLUTION = 616 +LANCE_VIDEO_VIT_DIVISIBLE_CROP = 28 +LANCE_VIDEO_BUCKET_STRIDE = 16 +LANCE_VIDEO_TEMPORAL_STRIDE = 4 # Wan2.2 VAE temporal downsample +LANCE_VIDEO_SAMPLE_FPS = 12 # upstream MultiClipsFrameSampler default +LANCE_VIDEO_MAX_DURATION = 6.0 # upstream max_duration (seconds) +LANCE_VIT_PATCH_SIZE = 14 # Qwen2.5-VL ViT spatial patch +LANCE_VIT_TEMPORAL_PATCH_SIZE = 2 # Qwen2.5-VL ViT temporal patch +LANCE_VIT_SPATIAL_MERGE = 2 # Qwen2.5-VL spatial merge factor +LANCE_VIT_NORM_MEAN = (0.48145466, 0.4578275, 0.40821073) +LANCE_VIT_NORM_STD = (0.26862954, 0.26130258, 0.27577711) + + +def get_3d_sincos_pos_embed_from_grid(embed_dim: int, grid: np.ndarray) -> np.ndarray: + """3-D sin-cos positional embedding (t, h, w), matching the upstream Lance + ``modeling/lance/modeling_utils.py`` dimension split exactly.""" + assert embed_dim % 2 == 0, "Embedding dimension must be even for 3D embeddings" + d = embed_dim // 3 + d = d if d % 2 == 0 else d - 1 + dim_t, dim_h = d, d + dim_w = embed_dim - 2 * d + assert dim_w % 2 == 0 + emb_t = get_1d_sincos_pos_embed_from_grid(dim_t, grid[0]) + emb_h = get_1d_sincos_pos_embed_from_grid(dim_h, grid[1]) + emb_w = get_1d_sincos_pos_embed_from_grid(dim_w, grid[2]) + return np.concatenate([emb_t, emb_h, emb_w], axis=1) + + +def get_3d_sincos_pos_embed(embed_dim: int, t: int, h: int, w: int) -> np.ndarray: + grid_t = np.arange(t, dtype=np.float32) + grid_h = np.arange(h, dtype=np.float32) + grid_w = np.arange(w, dtype=np.float32) + tt, hh, ww = np.meshgrid(grid_t, grid_h, grid_w, indexing="ij") + grid = np.stack([tt, hh, ww], axis=0) + return get_3d_sincos_pos_embed_from_grid(embed_dim, grid) + + +class LancePositionEmbedding3D(nn.Module): + """Frozen 3-D sin-cos latent position embedding for the video path. + + BAGEL only ships a 2-D ``PositionEmbedding`` (image latents). Lance's + ``Lance_3B_Video`` checkpoint adds a temporal axis; this mirrors upstream + ``modeling/lance/modeling_utils.py::PositionEmbedding3D``. The image path + uses ``t=1`` and is numerically equivalent to the 2-D embedding. + """ + + def __init__(self, max_num_frames: int, max_num_patch_per_side: int, hidden_size: int): + super().__init__() + self.max_num_frames = max_num_frames + self.max_num_patch_per_side = max_num_patch_per_side + self.hidden_size = hidden_size + n = max_num_frames * max_num_patch_per_side * max_num_patch_per_side + self.pos_embed = nn.Parameter(torch.zeros(n, hidden_size), requires_grad=False) + self._init_weights() + + def _init_weights(self) -> None: + pe = get_3d_sincos_pos_embed( + self.hidden_size, + self.max_num_frames, + self.max_num_patch_per_side, + self.max_num_patch_per_side, + ) + self.pos_embed.data.copy_(torch.from_numpy(pe).float()) + + def forward(self, position_ids: torch.Tensor) -> torch.Tensor: + return self.pos_embed[position_ids] + + +class LanceIdentityConnector(nn.Module): + """No-op connector for Lance. + + BAGEL's ``connector`` projects the ViT hidden size to the LLM hidden size. + Qwen2.5-VL's vision tower (which Lance uses) already projects to the LLM + hidden size internally via ``merger`` (``out_hidden_size = hidden_size``), + and the released Lance safetensors carry no ``connector.*`` weights. We + therefore plug in an ``Identity`` connector so ``forward_cache_update_vit`` + keeps its existing call site without a separate code path. + """ + + def forward(self, x: torch.Tensor) -> torch.Tensor: # noqa: D401 + return x + + +class LanceZeroVitPosEmbed(nn.Module): + """No-op positional embedding for Lance's ViT tokens. + + BAGEL adds an extra 2-D sin-cos ``vit_pos_embed`` on top of the ViT output. + Qwen2.5-VL's vision tower already carries its own (rotary) positional + encoding, and the released Lance safetensors carry no + ``vit_pos_embed.*`` weights. This module returns a broadcast-friendly + zero so the addition in ``forward_cache_update_vit`` is a no-op without + requiring a code-path branch. + """ + + def forward(self, position_ids: torch.Tensor) -> torch.Tensor: # noqa: D401 + return torch.zeros((), device=position_ids.device) + + +class LanceQwen2_5_VLNaViTWrapper(nn.Module): + """Packed (NaViT-style) wrapper around the Qwen2.5-VL vision tower. + + Bridges BAGEL's ``vit(packed_pixel_values, ...) -> [num_tokens, vit_hidden]`` + surface to the HF ``Qwen2_5_VisionTransformerPretrainedModel`` which + consumes ``(hidden_states, grid_thw)``. The packed call additionally needs + a per-image ``image_grid_thw`` so non-square images (and the + spatial-merge token count) line up — :class:`LanceBagel` stashes the grid + on the wrapper before invoking the ViT. + """ + + def __init__(self, vision_model: nn.Module, spatial_merge_size: int = 2): + super().__init__() + # Accept either the full Qwen2_5_VLForConditionalGeneration.visual or + # the bare vision transformer. + self.vision_model = getattr(vision_model, "visual", vision_model) + self.spatial_merge_size = spatial_merge_size + # Set by ``LanceBagel.forward_cache_update_vit`` before each call so + # the wrapper can pass true per-image (T, H, W) to the HF ViT. + self._pending_grid_thw: torch.Tensor | None = None + + @property + def config(self): # parity with SiglipNaViTWrapper.vision_model.config access + return self.vision_model.config + + def set_pending_grid_thw(self, grid_thw: torch.Tensor) -> None: + self._pending_grid_thw = grid_thw + + def forward( + self, + packed_pixel_values: torch.Tensor, + packed_flattened_position_ids: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: int, + ) -> torch.Tensor: + if self._pending_grid_thw is not None: + grid_thw_t = self._pending_grid_thw.to(packed_pixel_values.device) + self._pending_grid_thw = None + else: + # Fallback for square images when grid_thw was not pre-stashed. + cu = cu_seqlens.tolist() + grid_thw = [] + for i in range(len(cu) - 1): + n = cu[i + 1] - cu[i] + side = int(round(float(n) ** 0.5)) + grid_thw.append([1, side, side]) + grid_thw_t = torch.tensor(grid_thw, dtype=torch.long, device=packed_pixel_values.device) + + hidden = packed_pixel_values + if hasattr(self.vision_model, "dtype"): + hidden = hidden.to(self.vision_model.dtype) + out = self.vision_model(hidden_states=hidden, grid_thw=grid_thw_t) + # Qwen2.5-VL returns ``BaseModelOutputWithPooling(last_hidden_state=..., + # pooler_output=merged)``; the merged sequence (post spatial-merger, + # shape ``[sum_tokens_after_merge, out_hidden_size]``) is what the LLM + # consumes. + if hasattr(out, "pooler_output") and out.pooler_output is not None: + return out.pooler_output + if isinstance(out, tuple): + return out[0] + return out + + +class LanceBagel(Bagel): + """Bagel subclass with Lance-specific ViT handling. + + The released Lance checkpoint pairs BAGEL's Qwen2-MoT trunk with the + Qwen2.5-VL vision tower (whose ``merger`` already projects to LLM + ``hidden_size`` and which carries its own rotary positional encoding). + Two BAGEL assumptions therefore break for Lance: + + 1. ``prepare_vit_images`` calls ``transforms(image) -> (C, H, W)`` and + then ``patchify(...)``; Lance's image processor returns + ``(num_patches_flat, patch_features) + image_grid_thw`` already. + 2. ``forward_cache_update_vit`` adds a ``connector`` projection plus a + 2-D ``vit_pos_embed`` that has no checkpoint weights and would + also double-count the ViT's own positional encoding. + + This subclass overrides exactly those two methods. Everything else — + LLM trunk, VAE flow, generation loop, ``forward_cache_update_text`` / + ``forward_cache_update_vae`` — is reused unchanged. + """ + + @staticmethod + def _lance_compute_vit_buckets(): + """Replicate upstream Lance's ``BucketResize.init_buckets`` for the ViT path. + + Returns a list of ``((bucket_h, bucket_w), bucket_ratio)`` pairs in the + same order as :data:`LANCE_VIT_ASPECT_RATIOS`. Stays in sync with + upstream lance ``data/video/transforms/bucket_resize.py``. + """ + import math + + max_area = LANCE_VIT_BUCKET_RESOLUTION * LANCE_VIT_BUCKET_RESOLUTION + stride = LANCE_VIT_BUCKET_STRIDE + buckets: list[tuple[tuple[int, int], float]] = [] + for name in LANCE_VIT_ASPECT_RATIOS: + w_s, h_s = (int(v) for v in name.split(":")) + aspect = w_s / h_s + # Option 1: solve for width first. + rw1 = math.sqrt(max_area * aspect) + bw1 = round(rw1 / stride) * stride + bh1 = round(bw1 / aspect / stride) * stride + br1 = bw1 / bh1 + # Option 2: solve for height first. + rh2 = math.sqrt(max_area / aspect) + bh2 = round(rh2 / stride) * stride + bw2 = round(bh2 * aspect / stride) * stride + br2 = bw2 / bh2 + if abs(br1 - aspect) < abs(br2 - aspect): + bh, bw = bh1, bw1 + elif abs(br1 - aspect) > abs(br2 - aspect): + bh, bw = bh2, bw2 + else: + area1 = bh1 * bw1 + area2 = bh2 * bw2 + if abs(area1 - max_area) <= abs(area2 - max_area): + bh, bw = bh1, bw1 + else: + bh, bw = bh2, bw2 + buckets.append(((bh, bw), bw / bh)) + return buckets + + @staticmethod + def _lance_bucket_resize_pil(image): + """Apply upstream Lance's ViT preprocessing to a PIL image. + + Pipeline (matches lance-upstream's ``VideoTransform`` for the ViT path): + + 1. ``BucketResize``: pick the aspect-ratio bucket nearest to the input + and resize via ``RandomResizedCrop(scale=(1,1), ratio=fixed)``, + which for fixed ratio is a deterministic center-crop + resize. + 2. ``DivisibleCrop(28)``: center-crop to dims divisible by + ``patch_size * spatial_merge_size`` so the resulting ViT grid is + clean. + + Returns the post-pipeline PIL image (still RGB, just resized). Doing + this BEFORE the Qwen2VLImageProcessor means smart-resize is a no-op + and the ViT sees exactly upstream Lance's input. + """ + import numpy as _np + from torchvision.transforms import InterpolationMode, RandomResizedCrop + + if image.mode != "RGB": + image = image.convert("RGB") + w, h = image.size + img_ratio = w / h + + buckets = LanceBagel._lance_compute_vit_buckets() + # nearest bucket by absolute ratio difference + diffs = _np.array([abs(img_ratio - br) for _, br in buckets]) + bh, bw = buckets[int(diffs.argmin())][0] + bratio = bw / bh + + # RandomResizedCrop(scale=(1,1), ratio=(bratio, bratio)) is deterministic: + # picks the largest sub-image of the target aspect, then resizes to + # (bh, bw). We replicate this directly to avoid the torchvision + # transform's random-state dependency and to keep the result a PIL. + # IMPORTANT: upstream Lance wires BucketResize through ``NaResize`` + # which defaults to BICUBIC interpolation (NOT LANCZOS — BucketResize's + # own default is LANCZOS but ``NaResize`` overrides it). Using + # LANCZOS here gave byte-different ViT input pixels (max_diff~0.13) + # which fed into the noise-query attention as wrong K/V — symptom: + # vllm-omni preserved the input image's hat when upstream removed it. + rrc = RandomResizedCrop( + size=(bh, bw), + scale=(1.0, 1.0), + ratio=(bratio, bratio), + interpolation=InterpolationMode.BICUBIC, + ) + out = rrc(image) + # DivisibleCrop(28): center-crop to dims divisible by 28. + ow, oh = out.size + cropped_w = ow - (ow % LANCE_VIT_DIVISIBLE_CROP) + cropped_h = oh - (oh % LANCE_VIT_DIVISIBLE_CROP) + if cropped_w != ow or cropped_h != oh: + left = (ow - cropped_w) // 2 + top = (oh - cropped_h) // 2 + out = out.crop((left, top, left + cropped_w, top + cropped_h)) + return out + + # ------------------------------------------------------------------ # + # Video preprocessing — frame sampler + bucket resize for VAE / ViT + # ------------------------------------------------------------------ # + @staticmethod + def _lance_compute_video_buckets(resolution: int, stride: int): + """Generic version of :meth:`_lance_compute_vit_buckets` for arbitrary + ``resolution`` / ``stride`` (the video VAE uses 640/16, the video ViT + uses 616/16, both with the same 6 aspect-ratio table).""" + import math + + max_area = resolution * resolution + buckets: list[tuple[tuple[int, int], float]] = [] + for name in LANCE_VIT_ASPECT_RATIOS: + w_s, h_s = (int(v) for v in name.split(":")) + aspect = w_s / h_s + rw1 = math.sqrt(max_area * aspect) + bw1 = round(rw1 / stride) * stride + bh1 = round(bw1 / aspect / stride) * stride + br1 = bw1 / bh1 + rh2 = math.sqrt(max_area / aspect) + bh2 = round(rh2 / stride) * stride + bw2 = round(bh2 * aspect / stride) * stride + br2 = bw2 / bh2 + if abs(br1 - aspect) < abs(br2 - aspect): + bh, bw = bh1, bw1 + elif abs(br1 - aspect) > abs(br2 - aspect): + bh, bw = bh2, bw2 + else: + area1 = bh1 * bw1 + area2 = bh2 * bw2 + bh, bw = (bh1, bw1) if abs(area1 - max_area) <= abs(area2 - max_area) else (bh2, bw2) + buckets.append(((bh, bw), bw / bh)) + return buckets + + @staticmethod + def _lance_sample_frame_indices(num_input_frames: int, origin_fps: float) -> list[int]: + """Replicate upstream's ``MultiClipsFrameSampler`` (single-clip path). + + Upstream config (validation_dataset.py:110): + temporal=4, sample_fps=12, truncate=False, max_duration=6.0, + length_type="kn+1", assert_seconds=False. + + For a single clip covering the full input video this reduces to: + + duration = min(num_input_frames / origin_fps, max_duration) + n_frames = round(duration * sample_fps) + # round to k*temporal + 1 (or -temporal+1 if already a multiple) + if n_frames % temporal != 0: + n_frames = (n_frames // temporal) * temporal + 1 + else: + n_frames = (n_frames // temporal) * temporal + 1 - temporal + indices = np.linspace(0, num_input_frames - 1, n_frames, dtype=int) + + Verified against upstream for car (93 frames @ 24fps → 45 sampled) and + woman (121 frames @ 24fps → 57 sampled). + """ + import numpy as _np + + duration = num_input_frames / origin_fps + duration = min(duration, LANCE_VIDEO_MAX_DURATION) + n_frames = int(round(duration * LANCE_VIDEO_SAMPLE_FPS)) + t = LANCE_VIDEO_TEMPORAL_STRIDE + if n_frames % t != 0: + n_frames = (n_frames // t) * t + 1 + else: + n_frames = (n_frames // t) * t + 1 - t + n_frames = max(n_frames, 1) + return _np.linspace(0, num_input_frames - 1, n_frames, dtype=int).tolist() + + @staticmethod + def _lance_bucket_resize_video(frames_thwc, resolution: int, stride: int, divisible_crop: int, mean, std): + """Apply upstream Lance's VideoTransform pipeline to a (T,H,W,C) uint8 + ndarray. Returns ``(C, T, H', W')`` float tensor. + + Pipeline (mirrors lance-upstream's ``data/transforms.py::VideoTransform``): + + 1. BucketResize per PIL frame: RandomResizedCrop(scale=(1,1), + ratio=fixed-bucket-ratio, BICUBIC) → resized PIL → to_tensor. + Per-frame PIL path matches upstream's + ``BucketResize.__call__(list[PIL.Image])`` exactly — using a + batched tensor RRC instead drifted pixel values by ~0.13 + (in [0,1] range) because torchvision's tensor-BICUBIC kernel + differs subtly from PIL's BICUBIC kernel. + 2. DivisibleCrop(divisible_crop) — center-crop to dims divisible + by patch_size * spatial_merge_size (28) for ViT, or by VAE + downsample (16) for VAE. + 3. Normalize(mean, std) — VAE uses 0.5/0.5; ViT uses Qwen CLIP. + 4. Rearrange "t c h w -> c t h w" + """ + import numpy as _np + from PIL import Image as _Image + from torchvision.transforms import InterpolationMode, RandomResizedCrop + from torchvision.transforms.functional import center_crop as _ccrop + from torchvision.transforms.functional import to_tensor as _to_tensor + + if not isinstance(frames_thwc, _np.ndarray): + raise ValueError(f"Expected (T,H,W,C) uint8 ndarray; got {type(frames_thwc)}") + if frames_thwc.dtype != _np.uint8: + frames_thwc = frames_thwc.astype(_np.uint8) + if frames_thwc.ndim != 4 or frames_thwc.shape[-1] != 3: + raise ValueError(f"Expected (T,H,W,3) uint8; got {frames_thwc.shape}") + + T_in, H_in, W_in, _ = frames_thwc.shape + aspect = W_in / H_in + buckets = LanceBagel._lance_compute_video_buckets(resolution, stride) + diffs = _np.array([abs(aspect - br) for _, br in buckets]) + bh, bw = buckets[int(diffs.argmin())][0] + bratio = bw / bh + + rrc = RandomResizedCrop( + size=(bh, bw), + scale=(1.0, 1.0), + ratio=(bratio, bratio), + interpolation=InterpolationMode.BICUBIC, + ) + # Per-frame PIL → RRC → to_tensor → stack — matches upstream's + # ``BucketResize.__call__`` for ``list[PIL.Image]`` exactly. + per_frame = [] + for t in range(T_in): + img = _Image.fromarray(frames_thwc[t]) + img = rrc(img) + per_frame.append(_to_tensor(img)) # (C, bh, bw) float [0,1] + frames = torch.stack(per_frame, dim=0) # (T, C, bh, bw) + + # DivisibleCrop(divisible_crop) — center-crop along H/W. + _, C, Hb, Wb = frames.shape + Hc = Hb - (Hb % divisible_crop) + Wc = Wb - (Wb % divisible_crop) + if Hc != Hb or Wc != Wb: + frames = _ccrop(frames, [Hc, Wc]) + + # Normalize. Accept scalar or per-channel mean/std. + if isinstance(mean, (int, float)): + mean_t = torch.tensor([float(mean)] * C, dtype=frames.dtype).view(1, C, 1, 1) + else: + mean_t = torch.tensor(list(mean), dtype=frames.dtype).view(1, C, 1, 1) + if isinstance(std, (int, float)): + std_t = torch.tensor([float(std)] * C, dtype=frames.dtype).view(1, C, 1, 1) + else: + std_t = torch.tensor(list(std), dtype=frames.dtype).view(1, C, 1, 1) + frames = (frames - mean_t) / std_t + + # Rearrange "t c h w -> c t h w" + frames = frames.permute(1, 0, 2, 3).contiguous() + return frames + + @staticmethod + def _lance_patchify_vit_video(video_cthw: torch.Tensor): + """Replicate upstream ``patchify_video_with_merge``: (C, T, H, W) → + (num_patches, patch_dim). ``patch_dim = C * tp * p² = 3 * 2 * 14² = 1176``. + + Returns ``(packed_pixels, grid_thw)`` where ``grid_thw`` is the pre-merge + grid ``(T // tp, H // p, W // p)`` — same layout the Qwen2.5-VL ViT + expects (matches ``Qwen2VLImageProcessor.video_processor`` output up to + the actual pixel preprocessing). + """ + p = LANCE_VIT_PATCH_SIZE + tp = LANCE_VIT_TEMPORAL_PATCH_SIZE + ms = LANCE_VIT_SPATIAL_MERGE + # (C, T, H, W) -> (T, C, H, W) + video = video_cthw.permute(1, 0, 2, 3).contiguous() + T, C, H, W = video.shape + if T % tp != 0: + raise ValueError(f"ViT T={T} must be divisible by temporal_patch={tp}") + if H % p != 0 or W % p != 0: + raise ValueError(f"ViT H,W=({H},{W}) must be divisible by patch={p}") + gt, gh, gw = T // tp, H // p, W // p + if gh % ms != 0 or gw % ms != 0: + raise ValueError(f"ViT grid ({gh},{gw}) must be divisible by merge={ms}") + video = video.reshape(gt, tp, C, gh // ms, ms, p, gw // ms, ms, p) + # permute order matches upstream: (0,3,6,4,7,2,1,5,8) + video = video.permute(0, 3, 6, 4, 7, 2, 1, 5, 8) + patches = video.reshape(gt * gh * gw, C * tp * p * p) + grid_thw = torch.tensor([[gt, gh, gw]], dtype=torch.long) + return patches, grid_thw + + @classmethod + def _lance_video_preprocess(cls, frames_thwc, origin_fps: float): + """Full upstream-compatible video preprocessing for video_edit. + + Pipeline (mirrors lance-upstream ``validation_dataset.py`` for + ``--resolution video_480p``): + + 1. Frame-sample at 12 fps with max_duration=6s (k*temporal+1 rule). + 2. Bucket-resize the sampled frames TWICE: + - VAE branch (resolution=640, divisible=16, norm=0.5) + - ViT branch (resolution=616, divisible=28, norm=Qwen CLIP) + 3. ViT branch is patchified to (num_patches, 1176) for direct + consumption by Qwen2.5-VL. + + Returns: + vae_video — (C, T_lat_in, H_vae, W_vae) float tensor in + model-friendly [-1, 1] (mean=0.5, std=0.5). + vit_pixels — (num_patches, 1176) float tensor. + vit_grid_thw — (1, 3) long tensor with pre-merge grid (gt, gh, gw). + sampled_T — number of RGB frames after the temporal sampler. + """ + import numpy as _np + + if isinstance(frames_thwc, torch.Tensor): + frames_thwc = frames_thwc.detach().cpu().numpy() + if frames_thwc.dtype != _np.uint8: + # Allow [-1,1] / [0,1] float arrays; convert back to uint8 so the + # bucket transforms operate on the same domain as upstream. + arr = frames_thwc + if arr.max() <= 1.5: + arr = (arr * 255.0).clip(0, 255).astype(_np.uint8) + else: + arr = arr.astype(_np.uint8) + frames_thwc = arr + + T_in = frames_thwc.shape[0] + indices = cls._lance_sample_frame_indices(T_in, origin_fps) + sampled = frames_thwc[indices] # (T_sampled, H, W, 3) + + vae_video = cls._lance_bucket_resize_video( + sampled, + resolution=LANCE_VIDEO_VAE_RESOLUTION, + stride=LANCE_VIDEO_BUCKET_STRIDE, + divisible_crop=LANCE_VIDEO_VAE_DIVISIBLE_CROP, + mean=0.5, + std=0.5, + ) + vit_video = cls._lance_bucket_resize_video( + sampled, + resolution=LANCE_VIDEO_VIT_RESOLUTION, + stride=LANCE_VIDEO_BUCKET_STRIDE, + divisible_crop=LANCE_VIDEO_VIT_DIVISIBLE_CROP, + mean=LANCE_VIT_NORM_MEAN, + std=LANCE_VIT_NORM_STD, + ) + # Qwen2.5-VL ViT's temporal patch size is 2 → T must be even. + # Upstream pads by repeating the last frame for odd T (see + # ``validation_dataset.py:266``): + # ``video_tensor = torch.cat([video_tensor, last_frame], dim=1)`` + if vit_video.shape[1] % 2 == 1: + vit_video = torch.cat([vit_video, vit_video[:, -1:]], dim=1) + vit_pixels, vit_grid_thw = cls._lance_patchify_vit_video(vit_video) + return vae_video, vit_pixels, vit_grid_thw, len(indices) + + @staticmethod + def _qwen_vl_processor_call(processor, image): + """Invoke the Qwen2-VL image processor and return ``(pixel_values, grid_thw)``. + + Applies upstream Lance's BucketResize+DivisibleCrop BEFORE calling the + standard Qwen2VLImageProcessor. After the pre-resize the input dims + are already divisible by ``patch_size * spatial_merge_size`` and within + the processor's pixel bounds, so the processor's own smart-resize is a + no-op — the ViT sees the exact bucket-cropped pixels upstream uses. + """ + prepped = LanceBagel._lance_bucket_resize_pil(image) + proc_out = processor(images=prepped, return_tensors="pt") + pixel_values = proc_out["pixel_values"] + grid_thw = proc_out["image_grid_thw"] + return pixel_values, grid_thw + + # ------------------------------------------------------------------ # + # 3-D mRoPE plumbing + # ------------------------------------------------------------------ # + @staticmethod + def _mrope_broadcast(position_ids: torch.Tensor) -> torch.Tensor: + """Convert scalar ``(S,)`` position ids to mrope ``(3, S)`` by + broadcasting the scalar value to all three (t, h, w) axes. + + Lance/Qwen2.5-VL's mRoPE expects 3-D positions per token; for text + tokens (and image-edit context tokens) all three axes share the + same scalar position id. Vision-generation paths override this with + per-token ``(t, h, w)`` indices. + """ + if position_ids is None: + return position_ids + if position_ids.ndim == 1: + return position_ids.unsqueeze(0).expand(3, -1).contiguous() + return position_ids + + def prepare_prompts(self, *args, **kwargs): + gen_input, newlens, new_rope = super().prepare_prompts(*args, **kwargs) + if "packed_text_position_ids" in gen_input: + gen_input["packed_text_position_ids"] = self._mrope_broadcast(gen_input["packed_text_position_ids"]) + return gen_input, newlens, new_rope + + def _per_token_mrope_for_vae_latent(self, image_sizes, curr_rope) -> torch.Tensor: + """Build per-token Qwen2.5-VL mRoPE positions for the gen latent block. + + Matches upstream Lance (``Lance.validation_gen_KVcache`` → + ``get_rope_index``): + + start_of_image -> (P-1, P-1, P-1) + latent (hi, wi) -> (P, P+hi, P+wi) + end_of_image -> (P+max(h,w), ..., ...) + + where ``P = curr_position_id`` is the position counter AFTER the text + prefix (BAGEL parent already passes ``gen_context['ropes']``). The + ``-1`` on start_of_image matches upstream's get_rope_index layout: the + start marker sits one position before the latent block (which is + anchored at P), and the end marker sits ``max(h,w)`` positions past. + + Without per-token (h, w) variation, attention can't see the spatial + layout → image PSNR collapses ~6 dB after 30 steps. + """ + pos_t: list[int] = [] + pos_h: list[int] = [] + pos_w: list[int] = [] + for (H, W), P in zip(image_sizes, curr_rope): + h = H // self.latent_downsample + w = W // self.latent_downsample + # Upstream Lance (``shift_position_ids`` + ``get_rope_index``) + # layout for the gen latent block: + # start_of_image -> (P, P, P) + # latent (hi, wi) -> (P+1, P+1+hi, P+1+wi) + # end_of_image -> (P+max+1, P+max+1, P+max+1) + # where ``P = curr_position_id`` is the rope counter AFTER the + # text+ViT prefix (i.e., :attr:`gen_context['ropes']` snapshot + # taken before the VAE-ref prefill in ``_forward_image_edit``). + # This matches :meth:`_lance_native_prepare_vae_images` so the + # gen latent occupies the SAME rope range as the VAE-ref block + # (modality==1 / modality==2 share rope per upstream's + # ``i_sample_modality==2`` branch in ``shift_position_ids``). + max_hw = max(h, w) + pos_t.append(P) + pos_h.append(P) + pos_w.append(P) + for hi in range(h): + for wi in range(w): + pos_t.append(P + 1) + pos_h.append(P + 1 + hi) + pos_w.append(P + 1 + wi) + end_p = P + max_hw + 1 + pos_t.append(end_p) + pos_h.append(end_p) + pos_w.append(end_p) + return torch.stack( + [ + torch.tensor(pos_t, dtype=torch.long), + torch.tensor(pos_h, dtype=torch.long), + torch.tensor(pos_w, dtype=torch.long), + ], + dim=0, + ) + + def _per_token_mrope_for_video_latent(self, video_shapes, curr_rope) -> torch.Tensor: + """3-D analogue of :meth:`_per_token_mrope_for_vae_latent` for video. + + Matches upstream Lance's ``get_rope_index`` for ``modality==noise`` + (and ``modality==ref_source`` since they share rope per + ``shift_position_ids``). The temporal axis is amplified by + ``LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID`` so neighbouring + latent frames are well-separated, matching the upstream Qwen2.5-VL + rope convention. Verified against upstream's rope dump for + video_edit at video_480p (call09 cond noise iter 0): + + start_of_image -> (P, P, P) + latent (t,h,w) -> (P+1+t*2, P+1+h, P+1+w) + end_of_image -> (P+max+1, P+max+1, P+max+1) + + where ``max = max(t_lat*2-1, h_lat-1, w_lat-1) + 1`` matches the + end marker placement upstream uses (call09 final token at 159 for + T_lat=12, H_lat=35, W_lat=47 → max(22, 34, 46)+1=47, start=111, + end=158... actual upstream end=159, suggesting +2 from the body's + max). We mirror :meth:`_lance_native_prepare_vae_images` exactly + so the VAE_ref prefill and the gen noise share the same rope range. + """ + downsample_t = int(getattr(self.config.vae_config, "downsample_temporal", 4)) + t_scale = LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID + pos_t: list[int] = [] + pos_h: list[int] = [] + pos_w: list[int] = [] + for (T, H, W), P in zip(video_shapes, curr_rope): + h = H // self.latent_downsample + w = W // self.latent_downsample + t = (T - 1) // downsample_t + 1 + max_thw = max(int((t - 1) * t_scale), h - 1, w - 1) + 1 + pos_t.append(P) + pos_h.append(P) + pos_w.append(P) + for ti in range(t): + for hi in range(h): + for wi in range(w): + pos_t.append(P + 1 + int(ti * t_scale)) + pos_h.append(P + 1 + hi) + pos_w.append(P + 1 + wi) + end_p = P + max_thw + 1 + pos_t.append(end_p) + pos_h.append(end_p) + pos_w.append(end_p) + return torch.stack( + [ + torch.tensor(pos_t, dtype=torch.long), + torch.tensor(pos_h, dtype=torch.long), + torch.tensor(pos_w, dtype=torch.long), + ], + dim=0, + ) + + def prepare_vae_latent(self, curr_kvlens, curr_rope, image_sizes, new_token_ids): + gen_input = super().prepare_vae_latent(curr_kvlens, curr_rope, image_sizes, new_token_ids) + gen_input["packed_position_ids"] = self._per_token_mrope_for_vae_latent(image_sizes, curr_rope) + return gen_input + + def prepare_vae_latent_cfg(self, curr_kvlens, curr_rope, image_sizes): + gen_input = super().prepare_vae_latent_cfg(curr_kvlens, curr_rope, image_sizes) + gen_input["cfg_packed_position_ids"] = self._per_token_mrope_for_vae_latent(image_sizes, curr_rope) + return gen_input + + def prepare_start_tokens(self, *args, **kwargs): + gen_input = super().prepare_start_tokens(*args, **kwargs) + if isinstance(gen_input, dict): + for k in ( + "packed_query_position_ids", + "packed_text_position_ids", + "packed_position_ids", + ): + if k in gen_input and torch.is_tensor(gen_input[k]): + gen_input[k] = self._mrope_broadcast(gen_input[k]) + return gen_input + + def prepare_vit_videos( + self, + curr_kvlens, + curr_rope, + videos, + new_token_ids, + precomputed_vit=None, + ): + """Multi-frame ViT prefill for the ``x2t_video`` / ``video_edit`` paths. + + ``videos`` is a list of per-request video tensors / numpy arrays of + shape ``(T, H, W, 3)``. By default the Qwen2-VL video processor is + used to convert each video to ``(pixel_values_videos, video_grid_thw)``. + For ``video_edit`` precision matching, the pipeline may pre-compute the + upstream-style BucketResize output and pass it via ``precomputed_vit`` + — a list of ``(pixel_values, grid_thw)`` per video, in which case the + processor call is skipped. + """ + processor = getattr(self, "_lance_video_processor", None) + if processor is None and precomputed_vit is None: + raise RuntimeError( + "LanceBagel.prepare_vit_videos requires either ``precomputed_vit`` " + "or ``bagel._lance_video_processor`` (a Qwen2-VL-compatible video processor)." + ) + if precomputed_vit is not None and len(precomputed_vit) != len(videos): + raise ValueError( + f"precomputed_vit length ({len(precomputed_vit)}) must match number of videos ({len(videos)})" + ) + + packed_vit_token_indexes: list[int] = [] + vit_token_seqlens: list[int] = [] + packed_vit_tokens: list[torch.Tensor] = [] + packed_vit_position_ids: list[torch.Tensor] = [] + packed_text_ids: list[int] = [] + packed_text_indexes: list[int] = [] + packed_seqlens: list[int] = [] + packed_indexes: list[int] = [] + packed_key_value_indexes: list[int] = [] + grid_thw_list: list[torch.Tensor] = [] + per_axis_pos: list[tuple[int, int, int]] = [] + + merge_size = int(getattr(self.vit_model, "spatial_merge_size", 2)) + merge_factor = merge_size * merge_size + + _curr = curr = 0 + newlens: list[int] = [] + new_rope: list[int] = [] + for vi, (video, curr_kvlen, curr_position_id) in enumerate(zip(videos, curr_kvlens, curr_rope)): + packed_key_value_indexes.extend(range(curr, curr + curr_kvlen)) + curr += curr_kvlen + + packed_text_ids.append(new_token_ids["start_of_image"]) + packed_text_indexes.append(_curr) + packed_indexes.append(curr) + curr += 1 + _curr += 1 + per_axis_pos.append((curr_position_id, curr_position_id, curr_position_id)) + + if precomputed_vit is not None: + pixel_values, grid_thw = precomputed_vit[vi] + if not torch.is_tensor(grid_thw): + grid_thw = torch.as_tensor(grid_thw, dtype=torch.long) + if grid_thw.dim() == 1: + grid_thw = grid_thw.unsqueeze(0) + else: + proc_out = processor(videos=video, return_tensors="pt") + # Qwen2-VL video processor: pixel_values_videos is packed + # (num_patches_flat, patch_features) with a temporal axis + # ``T_lat`` in video_grid_thw. + pixel_values = proc_out["pixel_values_videos"] + grid_thw = proc_out["video_grid_thw"] + T_lat, H, W = (int(v) for v in grid_thw[0].tolist()) + num_patches_pre_merge = T_lat * H * W + assert num_patches_pre_merge == pixel_values.shape[0], ( + f"pixel_values rows ({pixel_values.shape[0]}) != T_lat*H*W ({num_patches_pre_merge}); " + f"video_grid_thw={grid_thw.tolist()}" + ) + num_vit_tokens = num_patches_pre_merge // merge_factor + + packed_vit_tokens.append(pixel_values) + vit_token_seqlens.append(num_patches_pre_merge) + packed_vit_position_ids.append(torch.arange(num_patches_pre_merge, dtype=torch.long)) + grid_thw_list.append(grid_thw[0].to(torch.long)) + + packed_vit_token_indexes.extend(range(_curr, _curr + num_vit_tokens)) + packed_indexes.extend(range(curr, curr + num_vit_tokens)) + curr += num_vit_tokens + _curr += num_vit_tokens + # Upstream Lance ViT layout (verified against video_edit rope dump + # for car @ video_480p — call2 t=(1000,1046), h,w=(64,110)): + # start_of_image -> (P, P, P) + # body (t,h,w) -> (P+1+t*ts, P+1+h, P+1+w) + # end_of_image -> (P+max+2, P+max+2, P+max+2) + # where max = max(ts*(T-1), h-1, w-1) + # then the entire ViT block's t-channel is shifted to 1000+ via + # ``shift_position_ids(pos_shift=1000)`` for modality=ref_vit. + h_merged = H // merge_size + w_merged = W // merge_size + t_scale = LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID + for ti in range(T_lat): + for hi in range(h_merged): + for wi in range(w_merged): + per_axis_pos.append( + ( + curr_position_id + 1 + int(ti * t_scale), + curr_position_id + 1 + hi, + curr_position_id + 1 + wi, + ) + ) + + packed_text_ids.append(new_token_ids["end_of_image"]) + packed_text_indexes.append(_curr) + packed_indexes.append(curr) + curr += 1 + _curr += 1 + max_offset = max(int((T_lat - 1) * t_scale), h_merged - 1, w_merged - 1) + end_p = curr_position_id + max_offset + 2 + per_axis_pos.append((end_p, end_p, end_p)) + + packed_seqlens.append(num_vit_tokens + 2) + newlens.append(curr_kvlen + num_vit_tokens + 2) + new_rope.append(end_p + 1) + + pos_t = torch.tensor([p[0] for p in per_axis_pos], dtype=torch.long) + pos_h = torch.tensor([p[1] for p in per_axis_pos], dtype=torch.long) + pos_w = torch.tensor([p[2] for p in per_axis_pos], dtype=torch.long) + # Apply upstream's ViT t-channel shift (modality=ref_vit=4 → +1000). + # Without this, the ViT block's K/V at the t-channel collides with the + # VAE_ref / noise block in mRoPE space (same low-100s range), and + # noise-query attention picks up cross-modal leakage. + pos_t = pos_t - pos_t[0] + 1000 + packed_position_ids_3d = torch.stack([pos_t, pos_h, pos_w], dim=0) + + generation_input = { + "packed_text_ids": torch.tensor(packed_text_ids, dtype=torch.long), + "packed_text_indexes": torch.tensor(packed_text_indexes, dtype=torch.long), + "vit_token_seqlens": torch.tensor(vit_token_seqlens, dtype=torch.int), + "packed_vit_tokens": torch.cat(packed_vit_tokens, dim=0), + "packed_vit_position_ids": torch.cat(packed_vit_position_ids, dim=0), + "packed_vit_token_indexes": torch.tensor(packed_vit_token_indexes, dtype=torch.long), + "packed_position_ids": packed_position_ids_3d, + "packed_seqlens": torch.tensor(packed_seqlens, dtype=torch.int), + "packed_indexes": torch.tensor(packed_indexes, dtype=torch.long), + "packed_key_value_indexes": torch.tensor(packed_key_value_indexes, dtype=torch.long), + "key_values_lens": torch.tensor(curr_kvlens, dtype=torch.int), + "_lance_grid_thw": torch.stack(grid_thw_list, dim=0), + } + return generation_input, newlens, new_rope + + def prepare_vit_images(self, curr_kvlens, curr_rope, images, transforms, new_token_ids): + # ``transforms`` is set up by ``BagelPipeline.forward`` to be + # ``processor(images=img, return_tensors='pt').pixel_values[0]``, which + # discards the ``image_grid_thw``. For Lance we need both, so we ignore + # the supplied lambda and re-call the processor here directly. Lance's + # pipeline always sets ``self.image_processor`` to ``Qwen2VLImageProcessor``. + if not hasattr(self, "_lance_image_processor"): + raise RuntimeError( + "LanceBagel.prepare_vit_images requires the pipeline to set " + "``bagel._lance_image_processor`` (a Qwen2-VL-compatible processor)." + ) + processor = self._lance_image_processor + + packed_vit_token_indexes: list[int] = [] + vit_token_seqlens: list[int] = [] + packed_vit_tokens: list[torch.Tensor] = [] + packed_vit_position_ids: list[torch.Tensor] = [] + packed_text_ids: list[int] = [] + packed_text_indexes: list[int] = [] + packed_seqlens: list[int] = [] + packed_indexes: list[int] = [] + packed_key_value_indexes: list[int] = [] + grid_thw_list: list[torch.Tensor] = [] + + merge_size = int(getattr(self.vit_model, "spatial_merge_size", 2)) + merge_factor = merge_size * merge_size + + _curr = curr = 0 + newlens: list[int] = [] + new_rope: list[int] = [] + # 3-D position ids per query token (mRoPE) — text framing tokens use + # scalar broadcast, image patches use per-token ``(P + t, P + h_merged, + # P + w_merged)`` so the Qwen2.5-VL backbone sees the spatial layout + # along the (h, w) axes of mRoPE. + per_axis_pos: list[tuple[int, int, int]] = [] + for image, curr_kvlen, curr_position_id in zip(images, curr_kvlens, curr_rope): + packed_key_value_indexes.extend(range(curr, curr + curr_kvlen)) + curr += curr_kvlen + + packed_text_ids.append(new_token_ids["start_of_image"]) + packed_text_indexes.append(_curr) + packed_indexes.append(curr) + curr += 1 + _curr += 1 + per_axis_pos.append((curr_position_id, curr_position_id, curr_position_id)) + + pixel_values, grid_thw = self._qwen_vl_processor_call(processor, image) + # pixel_values: (num_patches_flat, patch_features). grid_thw: (1, 3) with (T, H, W) in patch units. + T, H, W = (int(v) for v in grid_thw[0].tolist()) + num_patches_pre_merge = T * H * W + assert num_patches_pre_merge == pixel_values.shape[0], ( + f"pixel_values rows ({pixel_values.shape[0]}) != T*H*W ({num_patches_pre_merge}); " + f"image_grid_thw={grid_thw.tolist()}" + ) + num_img_tokens = num_patches_pre_merge // merge_factor + + packed_vit_tokens.append(pixel_values) + vit_token_seqlens.append(num_patches_pre_merge) + packed_vit_position_ids.append(torch.arange(num_patches_pre_merge, dtype=torch.long)) + grid_thw_list.append(grid_thw[0].to(torch.long)) + + packed_vit_token_indexes.extend(range(_curr, _curr + num_img_tokens)) + packed_indexes.extend(range(curr, curr + num_img_tokens)) + curr += num_img_tokens + _curr += num_img_tokens + # 3-D mRoPE positions for image patches. Upstream Lance + # (``modeling/lance/lance.py::get_rope_index`` + ``shift_position_ids``) + # places ViT body tokens at ``(P+1+t, P+1+h, P+1+w)`` and the + # surrounding markers at ``(P, ...)`` (start) / ``(P+max+1, ...)`` + # (end), then SHIFTS the t-channel of the entire ViT block to + # 1000+ (modality=ref_vit=4) so ViT keys don't conflate with VAE + # ref / noise keys in mRoPE space. ``new_rope = P + max + 2``. + h_merged = H // merge_size + w_merged = W // merge_size + max_hw = max(h_merged, w_merged) + for hi in range(h_merged): + for wi in range(w_merged): + per_axis_pos.append((curr_position_id + 1, curr_position_id + 1 + hi, curr_position_id + 1 + wi)) + + packed_text_ids.append(new_token_ids["end_of_image"]) + packed_text_indexes.append(_curr) + packed_indexes.append(curr) + curr += 1 + _curr += 1 + end_p = curr_position_id + max_hw + 1 + per_axis_pos.append((end_p, end_p, end_p)) + + packed_seqlens.append(num_img_tokens + 2) + newlens.append(curr_kvlen + num_img_tokens + 2) + new_rope.append(end_p + 1) + + pos_t = torch.tensor([p[0] for p in per_axis_pos], dtype=torch.long) + pos_h = torch.tensor([p[1] for p in per_axis_pos], dtype=torch.long) + pos_w = torch.tensor([p[2] for p in per_axis_pos], dtype=torch.long) + # Upstream Lance shifts the ViT block's t-channel into a far-away rope + # range (1000+) via ``shift_position_ids(pos_shift=1000)`` on + # modality=ref_vit (4). This makes ViT keys/queries mRoPE-orthogonal + # to noise/VAE-ref tokens along the t-channel, preventing leakage in + # the time-frequency dimension. We apply the same shift here so the + # ViT segment's K, V cache content matches upstream. + pos_t = pos_t - pos_t[0] + 1000 + packed_position_ids_3d = torch.stack([pos_t, pos_h, pos_w], dim=0) + + generation_input = { + "packed_text_ids": torch.tensor(packed_text_ids, dtype=torch.long), + "packed_text_indexes": torch.tensor(packed_text_indexes, dtype=torch.long), + "vit_token_seqlens": torch.tensor(vit_token_seqlens, dtype=torch.int), + "packed_vit_tokens": torch.cat(packed_vit_tokens, dim=0), + "packed_vit_position_ids": torch.cat(packed_vit_position_ids, dim=0), + "packed_vit_token_indexes": torch.tensor(packed_vit_token_indexes, dtype=torch.long), + "packed_position_ids": packed_position_ids_3d, + "packed_seqlens": torch.tensor(packed_seqlens, dtype=torch.int), + "packed_indexes": torch.tensor(packed_indexes, dtype=torch.long), + "packed_key_value_indexes": torch.tensor(packed_key_value_indexes, dtype=torch.long), + "key_values_lens": torch.tensor(curr_kvlens, dtype=torch.int), + # Lance-extra: per-image (T, H, W) so the ViT wrapper can pass the + # true grid to the HF tower (non-square images would otherwise be + # reconstructed incorrectly as ``sqrt(n)`` squares). + "_lance_grid_thw": torch.stack(grid_thw_list, dim=0), + } + return generation_input, newlens, new_rope + + def forward_cache_update_vit(self, *args, **kwargs): + # Pluck the Lance-specific extra and stash on the wrapper for the + # upcoming ViT call. Everything else delegates to the parent so we + # don't have to copy 60 lines of LLM-call boilerplate. + grid_thw = kwargs.pop("_lance_grid_thw", None) + if grid_thw is not None and hasattr(self.vit_model, "set_pending_grid_thw"): + self.vit_model.set_pending_grid_thw(grid_thw) + return super().forward_cache_update_vit(*args, **kwargs) + + def prepare_video_latent(self, curr_kvlens, curr_rope, video_shapes, new_token_ids): + """3-D analogue of :meth:`prepare_vae_latent`. + + ``video_shapes`` is a list of ``(T, H, W)`` per request (RGB pixel + space). We package one packed-init-noise tensor over ``T_lat × H_lat × + W_lat`` latent tokens per video, plus 1-D indices into the 3-D position + embedding table maintained by :class:`LancePositionEmbedding3D` + (`bagel.latent_pos_embed`). Latent geometry: + + - spatial: ``H_lat = H // latent_downsample`` (``=16`` for Lance) + - temporal: ``T_lat = (T - 1) // downsample_temporal + 1`` (``=4`` for Wan2.2) + - channels: ``latent_channel = 48`` + + Position ids are flattened ``t * max_per_side² + h * max_per_side + w`` + so they index directly into the ``(max_num_frames * max_per_side², + hidden_size)`` table. + """ + downsample_t = int(getattr(self.config.vae_config, "downsample_temporal", 4)) + max_per_side = int(self.max_latent_size) + + packed_text_ids, packed_text_indexes = [], [] + packed_vae_position_ids, packed_vae_token_indexes, packed_init_noises = [], [], [] + packed_seqlens, packed_indexes = [], [] + packed_key_value_indexes = [] + # 3-D position ids per token (mRoPE). ``per_axis_pos[i]`` is the list + # of (t, h, w) for the i-th query token; we stack into ``(3, S)`` at the + # end. Text framing tokens broadcast scalar to all axes; video latent + # tokens carry per-token ``(P + t, P + h, P + w)``. + per_axis_pos: list[tuple[int, int, int]] = [] + + query_curr = curr = 0 + for (T, H, W), curr_kvlen, curr_position_id in zip(video_shapes, curr_kvlens, curr_rope): + packed_key_value_indexes.extend(range(curr, curr + curr_kvlen)) + curr += curr_kvlen + + packed_text_ids.append(new_token_ids["start_of_image"]) + packed_text_indexes.append(query_curr) + packed_indexes.append(curr) + curr += 1 + query_curr += 1 + # Upstream Lance places gen-latent start_of_image at ``(P, P, P)`` + # (NOT ``P-1`` and NOT shifted to ViT t-range), body at + # ``(P+1+ti*t_scale, P+1+hi, P+1+wi)``, end at ``(P+max+1, ...)``. + # Verified against upstream's video_edit rope dump for T_lat=12, + # H_lat=35, W_lat=47 (call09 / cond noise iter 0). + per_axis_pos.append((curr_position_id, curr_position_id, curr_position_id)) + + h = H // self.latent_downsample + w = W // self.latent_downsample + t = (T - 1) // downsample_t + 1 + num_video_tokens = t * h * w + + # 1-D position ids into the 3-D table (frame-major, then row, then col). + tt, hh, ww = torch.meshgrid(torch.arange(t), torch.arange(h), torch.arange(w), indexing="ij") + tt_flat = tt.flatten().tolist() + hh_flat = hh.flatten().tolist() + ww_flat = ww.flatten().tolist() + vae_position_ids = (tt * (max_per_side * max_per_side) + hh * max_per_side + ww).flatten() + packed_vae_position_ids.append(vae_position_ids) + + packed_init_noises.append(torch.randn(num_video_tokens, self.latent_channel * self.latent_patch_size**2)) + + packed_vae_token_indexes.extend(range(query_curr, query_curr + num_video_tokens)) + packed_indexes.extend(range(curr, curr + num_video_tokens)) + curr += num_video_tokens + query_curr += num_video_tokens + # Per-token 3-D mRoPE positions, matching upstream Lance's + # ``get_rope_index``: the temporal axis is amplified by + # ``tokens_per_second * second_per_grid_t`` (see Qwen2.5-VL docs) + # so neighbouring latent frames are well-separated in rope space. + # ``second_per_grid_t = 1.0`` matches upstream's default; + # ``tokens_per_second = 2`` is Lance's vit config. + t_scale = LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID + for ti, hi, wi in zip(tt_flat, hh_flat, ww_flat): + per_axis_pos.append( + ( + curr_position_id + 1 + int(ti * t_scale), + curr_position_id + 1 + hi, + curr_position_id + 1 + wi, + ) + ) + + packed_text_ids.append(new_token_ids["end_of_image"]) + packed_text_indexes.append(query_curr) + packed_indexes.append(curr) + curr += 1 + query_curr += 1 + # end_of_image sits "past" the latent block at ``P+max+1`` on every + # axis, where ``max = max(t_scale*(t-1), h-1, w-1) + 1``. Verified + # against upstream's call09 final token at (159, 159, 159) for + # T_lat=12 (t_scale=2 → max_t=22), H_lat=35 (max_h=34), W_lat=47 + # (max_w=46), with P=111: end_p = 111 + 46 + 1 + 1 = 159. ✓ + max_thw = max(int((t - 1) * t_scale), h - 1, w - 1) + 1 + end_p = curr_position_id + max_thw + 1 + per_axis_pos.append((end_p, end_p, end_p)) + + packed_seqlens.append(num_video_tokens + 2) + + # Stack into (3, total_query_tokens) for mRoPE. + pos_t = torch.tensor([p[0] for p in per_axis_pos], dtype=torch.long) + pos_h = torch.tensor([p[1] for p in per_axis_pos], dtype=torch.long) + pos_w = torch.tensor([p[2] for p in per_axis_pos], dtype=torch.long) + packed_position_ids_3d = torch.stack([pos_t, pos_h, pos_w], dim=0) + + generation_input = { + "packed_text_ids": torch.tensor(packed_text_ids, dtype=torch.long), + "packed_text_indexes": torch.tensor(packed_text_indexes, dtype=torch.long), + "packed_init_noises": torch.cat(packed_init_noises, dim=0), + "packed_vae_position_ids": torch.cat(packed_vae_position_ids, dim=0), + "packed_vae_token_indexes": torch.tensor(packed_vae_token_indexes, dtype=torch.long), + "packed_seqlens": torch.tensor(packed_seqlens, dtype=torch.int), + "packed_position_ids": packed_position_ids_3d, + "key_values_lens": torch.tensor(curr_kvlens, dtype=torch.int), + "packed_indexes": torch.tensor(packed_indexes, dtype=torch.long), + "packed_key_value_indexes": torch.tensor(packed_key_value_indexes, dtype=torch.long), + } + return generation_input + + def prepare_video_latent_cfg(self, curr_kvlens, curr_rope, video_shapes): + """3-D analogue of :meth:`prepare_vae_latent_cfg` (CFG side). + + Mirrors :meth:`prepare_video_latent`'s mRoPE 3-D position layout + EXACTLY, including the ``LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID`` + temporal scaling. Without that scaling, the cfg_text branch attends + with different rope coordinates than the cond branch (and than + upstream's ``get_rope_index``), which makes ``cfg_text_v_t`` diverge + from upstream and the CFG combination amplifies the error every + denoise step. + """ + downsample_t = int(getattr(self.config.vae_config, "downsample_temporal", 4)) + + packed_indexes, packed_key_value_indexes = [], [] + per_axis_pos: list[tuple[int, int, int]] = [] + query_curr = curr = 0 + t_scale = LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID + for (T, H, W), curr_kvlen, curr_position_id in zip(video_shapes, curr_kvlens, curr_rope): + packed_key_value_indexes.extend(range(curr, curr + curr_kvlen)) + curr += curr_kvlen + + packed_indexes.append(curr) + curr += 1 + query_curr += 1 + # Match :meth:`prepare_video_latent` convention exactly (see + # comments there for the upstream rope layout this replicates). + per_axis_pos.append((curr_position_id, curr_position_id, curr_position_id)) + + h = H // self.latent_downsample + w = W // self.latent_downsample + t = (T - 1) // downsample_t + 1 + num_video_tokens = t * h * w + packed_indexes.extend(range(curr, curr + num_video_tokens)) + curr += num_video_tokens + query_curr += num_video_tokens + tt, hh, ww = torch.meshgrid(torch.arange(t), torch.arange(h), torch.arange(w), indexing="ij") + for ti, hi, wi in zip(tt.flatten().tolist(), hh.flatten().tolist(), ww.flatten().tolist()): + per_axis_pos.append( + ( + curr_position_id + 1 + int(ti * t_scale), + curr_position_id + 1 + hi, + curr_position_id + 1 + wi, + ) + ) + + packed_indexes.append(curr) + curr += 1 + query_curr += 1 + max_thw = max(int((t - 1) * t_scale), h - 1, w - 1) + 1 + end_p = curr_position_id + max_thw + 1 + per_axis_pos.append((end_p, end_p, end_p)) + + pos_t = torch.tensor([p[0] for p in per_axis_pos], dtype=torch.long) + pos_h = torch.tensor([p[1] for p in per_axis_pos], dtype=torch.long) + pos_w = torch.tensor([p[2] for p in per_axis_pos], dtype=torch.long) + cfg_packed_position_ids_3d = torch.stack([pos_t, pos_h, pos_w], dim=0) + + return { + "cfg_packed_position_ids": cfg_packed_position_ids_3d, + "cfg_packed_query_indexes": torch.tensor(packed_indexes, dtype=torch.long), + "cfg_key_values_lens": torch.tensor(curr_kvlens, dtype=torch.int), + "cfg_packed_key_value_indexes": torch.tensor(packed_key_value_indexes, dtype=torch.long), + } + + # ------------------------------------------------------------------ # + # Image-edit / video-edit VAE prefill + # ------------------------------------------------------------------ # + def _lance_native_prepare_vae_images( + self, + curr_kvlens, + curr_rope, + images, + transforms, + new_token_ids, + timestep=0, + is_video: bool = False, + ): + """Lance-native ``prepare_vae_images`` with 3-D mRoPE positions. + + Mirrors :meth:`Bagel.prepare_vae_images` but emits per-token mRoPE + ``(curr + t, curr + h, curr + w)`` positions for VAE latent tokens + (BAGEL emits scalar ``curr_position_id`` for all of them, which + triggers a CUDA index-out-of-bounds gather on Lance's Qwen2.5-VL + backbone). For still images ``t = 0``; for video clips ``t`` + ranges over ``T_lat = (T - 1) // downsample_temporal + 1``. + """ + downsample_t = int(getattr(self.config.vae_config, "downsample_temporal", 4)) + max_per_side = int(self.max_latent_size) + + packed_text_ids: list[int] = [] + packed_text_indexes: list[int] = [] + packed_vae_position_ids: list[torch.Tensor] = [] + packed_vae_token_indexes: list[int] = [] + packed_indexes: list[int] = [] + packed_key_value_indexes: list[int] = [] + packed_seqlens: list[int] = [] + patchified_vae_latent_shapes: list[tuple] = [] + vae_image_tensors: list[torch.Tensor] = [] + per_axis_pos: list[tuple[int, int, int]] = [] + + query_curr = curr = 0 + newlens: list[int] = [] + new_rope: list[int] = [] + for image, curr_kvlen, curr_position_id in zip(images, curr_kvlens, curr_rope): + packed_key_value_indexes.extend(range(curr, curr + curr_kvlen)) + curr += curr_kvlen + + packed_text_ids.append(new_token_ids["start_of_image"]) + packed_text_indexes.append(query_curr) + packed_indexes.append(curr) + curr += 1 + query_curr += 1 + per_axis_pos.append((curr_position_id, curr_position_id, curr_position_id)) + + image_tensor = transforms(image) + vae_image_tensors.append(image_tensor) + # ``image_tensor`` is ``(C, H, W)`` for image input or ``(C, T, H, W)`` + # for video input. We derive the *latent* grid from the RGB spatial + # dims and our Wan2.2 VAE downsample factor. + if image_tensor.dim() == 3: + H, W = image_tensor.shape[1:] + T = 1 + elif image_tensor.dim() == 4: + _, T, H, W = image_tensor.shape + else: + raise ValueError(f"vae transforms must return 3-D or 4-D tensor; got {image_tensor.shape}") + h_lat = H // self.latent_downsample + w_lat = W // self.latent_downsample + t_lat = ((T - 1) // downsample_t + 1) if is_video else 1 + patchified_vae_latent_shapes.append((t_lat, h_lat, w_lat) if is_video else (h_lat, w_lat)) + + # 1-D index into the (3-D when video) ``latent_pos_embed`` table. + tt, hh, ww = torch.meshgrid(torch.arange(t_lat), torch.arange(h_lat), torch.arange(w_lat), indexing="ij") + vae_pos_ids = (tt * (max_per_side * max_per_side) + hh * max_per_side + ww).flatten() + packed_vae_position_ids.append(vae_pos_ids) + + num_image_tokens = t_lat * h_lat * w_lat + packed_vae_token_indexes.extend(range(query_curr, query_curr + num_image_tokens)) + packed_indexes.extend(range(curr, curr + num_image_tokens)) + curr += num_image_tokens + query_curr += num_image_tokens + # Upstream Lance places VAE-ref body at ``(P+1+t*t_scale, P+1+h, P+1+w)`` + # with the start marker at ``(P, P, P)`` and the end marker at + # ``(P+max+1, ...)``; gen-latent tokens (noise) share the SAME + # rope range via the modality_map==1/2 trick, so this is the + # convention used by :meth:`_per_token_mrope_for_vae_latent` (2D) + # and :meth:`_per_token_mrope_for_video_latent` (3D) below. + # For video, ``t_scale = LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID`` + # matches upstream Qwen2.5-VL's temporal-axis scaling; for still + # images we leave the t-channel scalar (T=1 → ``ti=0`` anyway). + t_scale = LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID if is_video else 1 + max_thw = max(int((t_lat - 1) * t_scale), h_lat - 1, w_lat - 1) + 1 + for ti, hi, wi in zip(tt.flatten().tolist(), hh.flatten().tolist(), ww.flatten().tolist()): + per_axis_pos.append( + ( + curr_position_id + 1 + int(ti * t_scale), + curr_position_id + 1 + hi, + curr_position_id + 1 + wi, + ) + ) + + packed_text_ids.append(new_token_ids["end_of_image"]) + packed_text_indexes.append(query_curr) + packed_indexes.append(curr) + curr += 1 + query_curr += 1 + end_p = curr_position_id + max_thw + 1 + per_axis_pos.append((end_p, end_p, end_p)) + + packed_seqlens.append(num_image_tokens + 2) + newlens.append(curr_kvlen + num_image_tokens + 2) + new_rope.append(end_p + 1) + + image_sizes = [item.shape for item in vae_image_tensors] + max_image_size = [max(item) for item in list(zip(*image_sizes))] + padded_images = torch.zeros(size=(len(vae_image_tensors), *max_image_size)) + for i, image_tensor in enumerate(vae_image_tensors): + if image_tensor.dim() == 3: + padded_images[i, :, : image_tensor.shape[1], : image_tensor.shape[2]] = image_tensor + else: + padded_images[i, :, : image_tensor.shape[1], : image_tensor.shape[2], : image_tensor.shape[3]] = ( + image_tensor + ) + + pos_t = torch.tensor([p[0] for p in per_axis_pos], dtype=torch.long) + pos_h = torch.tensor([p[1] for p in per_axis_pos], dtype=torch.long) + pos_w = torch.tensor([p[2] for p in per_axis_pos], dtype=torch.long) + packed_position_ids_3d = torch.stack([pos_t, pos_h, pos_w], dim=0) + + generation_input = { + "padded_images": padded_images, + "patchified_vae_latent_shapes": patchified_vae_latent_shapes, + "packed_vae_position_ids": torch.cat(packed_vae_position_ids, dim=0), + "packed_timesteps": torch.tensor([timestep]), + "packed_vae_token_indexes": torch.tensor(packed_vae_token_indexes, dtype=torch.long), + "packed_text_ids": torch.tensor(packed_text_ids, dtype=torch.long), + "packed_text_indexes": torch.tensor(packed_text_indexes, dtype=torch.long), + "packed_position_ids": packed_position_ids_3d, + "packed_seqlens": torch.tensor(packed_seqlens, dtype=torch.int), + "packed_indexes": torch.tensor(packed_indexes, dtype=torch.long), + "packed_key_value_indexes": torch.tensor(packed_key_value_indexes, dtype=torch.long), + "key_values_lens": torch.tensor(curr_kvlens, dtype=torch.int), + } + return generation_input, newlens, new_rope + + def prepare_vae_images(self, curr_kvlens, curr_rope, images, transforms, new_token_ids, timestep=0): + """VAE prefill router. + + - When ``images`` is non-empty (image_edit / video_edit path on the + image side): delegate to :meth:`_lance_native_prepare_vae_images` + which emits Lance's 3-D mRoPE positions and a real VAE prefill, + letting BAGEL's parent ``image_edit`` flow handle the rest. + - When ``images`` is empty (t2i / x2t paths): short-circuit with the + no-op output, mirroring BAGEL's "no image to prefill" sentinel. + """ + if images: + return self._lance_native_prepare_vae_images( + curr_kvlens=curr_kvlens, + curr_rope=curr_rope, + images=images, + transforms=transforms, + new_token_ids=new_token_ids, + timestep=timestep, + is_video=False, + ) + generation_input = { + "padded_images": torch.empty(0, 3, 0, 0), + "patchified_vae_latent_shapes": [], + "packed_vae_position_ids": torch.empty(0, dtype=torch.long), + "packed_timesteps": torch.tensor([timestep]), + "packed_vae_token_indexes": torch.empty(0, dtype=torch.long), + "packed_text_ids": torch.empty(0, dtype=torch.long), + "packed_text_indexes": torch.empty(0, dtype=torch.long), + "packed_position_ids": torch.empty(0, dtype=torch.long), + "packed_seqlens": torch.empty(0, dtype=torch.int), + "packed_indexes": torch.empty(0, dtype=torch.long), + "packed_key_value_indexes": torch.empty(0, dtype=torch.long), + "key_values_lens": torch.tensor(curr_kvlens, dtype=torch.int), + } + return generation_input, list(curr_kvlens), list(curr_rope) + + def forward_cache_update_vae( + self, + vae_model, + past_key_values, + padded_images=None, + patchified_vae_latent_shapes=None, + packed_vae_position_ids=None, + packed_timesteps=None, + packed_vae_token_indexes=None, + packed_text_ids=None, + packed_text_indexes=None, + packed_position_ids=None, + packed_seqlens=None, + packed_indexes=None, + key_values_lens=None, + packed_key_value_indexes=None, + precomputed_latent=None, + ): + """Lance-native VAE prefill that *actually* scatters the encoded + latents into the LLM query sequence. + + :meth:`Bagel.forward_cache_update_vae` in vllm-omni computes + ``packed_latent = vae2llm(...) + time_embed + pos_embed`` and then + passes only ``packed_text_ids`` to the LLM — the VAE embeddings + never enter the query sequence (the LLM builds it from + ``embed_tokens(packed_text_ids)`` which is just the 2 framing + tokens). The mismatch between ``query_lens = [num_vae + 2]`` and + the resulting 2-token sequence is what crashes the gather inside + attention. + + We scatter both pieces explicitly: text framing tokens at + ``packed_text_indexes`` and the VAE latent embeddings at + ``packed_vae_token_indexes``, producing a full-length + ``(sum(packed_seqlens), hidden)`` sequence the LLM can attend + over. Empty prep data (legacy x2t / x2t_video no-op path) is + short-circuited. + """ + if ( + packed_text_ids is None + or packed_text_ids.numel() == 0 + or (precomputed_latent is None and (padded_images is None or padded_images.numel() == 0)) + ): + return past_key_values + + # ``vae_model.encode`` samples from the posterior via + # ``mu + std * randn_like(std)`` (Wan2.2 VAE's ``reparameterize`` path + # — both upstream and vllm-omni default to ``use_sample=True``). Each + # call therefore returns a DIFFERENT latent. Upstream's image_edit + # encodes the reference image ONCE and reuses the resulting latent + # for both the cond and cfg_text branches; calling encode separately + # per branch made vllm-omni's gen vs cfg VAE-ref K cache diverge by + # ~6% rel_l2, which cascaded into ~33% v_t divergence at iter 0 on + # the cond branch (cfg matched within bf16 noise because it sees the + # SAME ref VAE K in both implementations — only gen-vs-cfg drift + # matters for the qualitative edit, since the noise query attends to + # both branches' caches at every iter). The pipeline now encodes + # the ref once and threads the encoded latent into both prefill + # calls via ``precomputed_latent``. + if precomputed_latent is not None: + padded_latent = precomputed_latent + else: + padded_latent = vae_model.encode(padded_images) + p = self.latent_patch_size + packed_latent_list = [] + for latent, shape in zip(padded_latent, patchified_vae_latent_shapes): + if isinstance(shape, tuple) and len(shape) == 3: + t_lat, h_lat, w_lat = shape + latent = latent[:, :t_lat, : h_lat * p, : w_lat * p].reshape( + self.latent_channel, t_lat, h_lat, p, w_lat, p + ) + latent = torch.einsum("cthpwq->thwpqc", latent).reshape(-1, p * p * self.latent_channel) + else: + h_lat, w_lat = shape + latent = latent[:, : h_lat * p, : w_lat * p].reshape(self.latent_channel, h_lat, p, w_lat, p) + latent = torch.einsum("chpwq->hwpqc", latent).reshape(-1, p * p * self.latent_channel) + packed_latent_list.append(latent) + packed_latent = torch.cat(packed_latent_list, dim=0) + + packed_pos_embed = self.latent_pos_embed(packed_vae_position_ids) + packed_timestep_embeds = self.time_embedder(packed_timesteps) + packed_latent = self.vae2llm(packed_latent) + packed_timestep_embeds + packed_pos_embed + + # ---- Scatter text + VAE embeds into the full query sequence ---- # + total_len = int(packed_seqlens.sum().item()) + packed_text_embed = self.language_model.model.embed_tokens(packed_text_ids) + packed_query_sequence = packed_latent.new_zeros((total_len, packed_latent.shape[-1])) + packed_query_sequence[packed_text_indexes] = packed_text_embed.to(packed_query_sequence.dtype) + packed_query_sequence[packed_vae_token_indexes] = packed_latent.to(packed_query_sequence.dtype) + + # Upstream Lance routes the VAE-ref block through MoE_GEN because + # ``packed_gen_token_indexes`` in ``Lance.validation_gen_KVcache`` + # is set to ``current_vae_token_indexes_local`` — i.e., it includes + # BOTH ref VAE tokens (modality=2 ref_source) AND noise tokens + # (modality=1). For the VAE-ref segment forward, the mask + # ``packed_gen_token_indexes ∈ [current_cond_start, current_cond_end)`` + # filters in the VAE-ref body tokens → ``mode_ = "gen"``. + extra_inputs = {} + if self.use_moe: + extra_inputs = { + "mode": "gen", + "packed_vae_token_indexes": packed_vae_token_indexes, + "packed_text_indexes": packed_text_indexes, + } + + # ``Qwen2MoTForCausalLM.forward`` post-main-merge no longer accepts + # ``packed_query_indexes`` / ``key_values_lens`` / + # ``packed_key_value_indexes`` — derived internally now. + output = self.language_model.forward( + packed_query_sequence=packed_query_sequence, + query_lens=packed_seqlens, + packed_query_position_ids=packed_position_ids, + past_key_values=past_key_values, + update_past_key_values=True, + is_causal=False, + **extra_inputs, + ) + return output.past_key_values diff --git a/vllm_omni/diffusion/models/lance/pipeline_lance.py b/vllm_omni/diffusion/models/lance/pipeline_lance.py new file mode 100644 index 000000000..5639e9261 --- /dev/null +++ b/vllm_omni/diffusion/models/lance/pipeline_lance.py @@ -0,0 +1,1935 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""LancePipeline — Lance (ByteDance) packaged for the vLLM-Omni diffusion engine. + +Lance is BAGEL-lineage (Qwen2-MoT unified AR+diffusion), so the transformer +core and the entire generation/forward machinery are inherited unchanged from +:class:`BagelPipeline`. Only model *construction* differs, and only in three +well-localized places: + + 1. Checkpoint layout — the HF repo ``bytedance-research/Lance`` bundles + ``Lance_3B/`` (image) and ``Lance_3B_Video/`` (video) LLM checkpoints, + ``Qwen2.5-VL-ViT/`` (understanding ViT) and ``Wan2.2_VAE.pth`` (VAE) in a + single repo. There is no BAGEL-style top-level ``config.json`` carrying + ``vae_config`` / ``vit_config`` / ``latent_patch_size`` — those are Lance + constants taken from upstream ``config/config_factory.py`` and + ``inference_lance.sh`` and hardcoded in :data:`LANCE_DEFAULTS`. + 2. Understanding ViT — Qwen2.5-VL vision tower (bundled + ``Qwen2.5-VL-ViT/vit.safetensors``) instead of SigLIP. + 3. VAE — Wan2.2 (``Wan2.2_VAE.pth``) instead of the BAGEL autoencoder. + +Scope: this lands the **image path** (``t2i`` / ``image_edit`` / ``x2t_image``) +which is the direct BAGEL analogue. The ``Lance_3B_Video`` path needs the 3-D +latent position embedding (:class:`LancePositionEmbedding3D`) and temporal VAE +handling and is an explicit follow-up — see the PR description. + +Bring-up status (verified on a B300 against ``bytedance-research/Lance``): + * t2i: end-to-end working (1024x1024 image in ~6 s, 0 missing keys). + * x2t (image understanding): plumbing wired (Qwen2.5-VL ViT + Qwen2-VL + image processor + no-op connector/vit_pos_embed; VAE prefill skipped + for Lance via :class:`LanceBagel`). The pipeline runs to completion + without crashes but currently emits an immediate EOS because the + Qwen2.5-VL backbone needs mRoPE position ids on the image+text + sequence and we presently force scalar positions + (``rope_scaling = None``); enabling mRoPE end-to-end is the next + follow-up. + * image_edit (img2img): blocked on the same VAE-prefill issue + (Wan2.2 latents do not map onto BAGEL's ``latent_pos_embed`` grid); + needs a Lance-specific ``prepare_vae_images``. + * video (``Lance_3B_Video``): needs the 3-D latent position embedding + wired into ``Bagel`` plus a multi-frame VAE decode path in the + pipeline. :class:`LanceWanVAE.decode_video` and + :class:`LancePositionEmbedding3D` are already in place. + * Two-stage (AR thinker + DiT): needs ``LanceConfig`` / ``LanceProcessor`` + registered in the ``vllm`` package; tracked in a separate PR. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass + +import torch +from PIL import Image +from vllm.logger import init_logger + +from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.diffusion.models.bagel.pipeline_bagel import ( + BagelPipeline, + add_special_tokens, +) +from vllm_omni.model_executor.model_loader.weight_utils import ( + download_weights_from_hf_specific, +) + +from .lance_transformer import ( + LanceBagel, + LanceIdentityConnector, + LancePositionEmbedding3D, + LanceQwen2_5_VLNaViTWrapper, + LanceZeroVitPosEmbed, + Qwen2MoTConfig, + Qwen2MoTForCausalLM, +) +from .prompts import SYSTEM_PROMPTS +from .wan_vae import LanceWanVAE + +logger = init_logger(__name__) + + +def _extract_user_instruction(rendered: str) -> str: + """Pull just the user instruction out of a chat-template-rendered prompt. + + ``render_lance_prompt`` wraps the instruction in: + + <|im_start|>system\\n{sys}<|im_end|>\\n + <|im_start|>user\\n{vision_block}{user_text}<|im_end|>\\n + <|im_start|>assistant\\n + + The ``image_edit`` pipeline rebuilds the chat template segment-by-segment + (so the rope positions and segment boundaries match upstream Lance), and + therefore needs only ``user_text`` — not the wrapped string. This helper + extracts it; if the input doesn't look templated it's returned as-is. + """ + if "<|im_start|>user" not in rendered: + return rendered.strip() + # User block sits between the user header and the next ``<|im_end|>``. + after_user = rendered.split("<|im_start|>user", 1)[1] + after_user = after_user.lstrip("\n") + user_block = after_user.split("<|im_end|>", 1)[0] + # Strip any vision_start..vision_end blocks. + out = [] + i = 0 + while i < len(user_block): + if user_block.startswith("<|vision_start|>", i): + j = user_block.find("<|vision_end|>", i) + if j == -1: + break + i = j + len("<|vision_end|>") + continue + out.append(user_block[i]) + i += 1 + return "".join(out).strip() + + +@dataclass(frozen=True) +class LanceDefaults: + """Lance constants that upstream keeps in ``config/config_factory.py`` / + ``inference_lance.sh`` rather than in any shipped JSON. + + Verified against ``bytedance-research/Lance``'s released ``Lance_3B/model.safetensors``: + ``vae2llm.weight = (2048, 48)`` ⇒ ``patch_latent_dim = latent_patch_size**2 * z_channels = 1 * 48``, + i.e. Lance does *not* unfold the Wan latent into a 2×2 patch the way BAGEL does (Wan2.2 already + patchifies internally), and ``latent_pos_embed.pos_embed = (4096, 2048)`` ⇒ ``max_latent_size = 64``. + """ + + # latent / patch geometry (pt, ph, pw); image path only uses the spatial 1. + latent_patch_size_spatial: int = 1 + latent_patch_size_temporal: int = 1 + max_latent_size: int = 64 + # Lance_3B_Video ships ``latent_pos_embed.pos_embed`` of shape + # ``(31 * 64 * 64, 2048) = (126976, 2048)`` ⇒ max_num_frames in latent + # space is 31, equivalent to ``(num_frames - 1) // downsample_temporal + 1`` + # for up to 121 RGB frames. + max_num_video_latent_frames: int = 31 + vit_max_num_patch_per_side: int = 70 + connector_act: str = "gelu_pytorch_tanh" + timestep_shift: float = 3.5 + num_timesteps: int = 30 + cfg_text_scale: float = 4.0 + # Wan2.2 VAE + vae_z_channels: int = 48 + vae_downsample_spatial: int = 16 + vae_downsample_temporal: int = 4 + + +LANCE_DEFAULTS = LanceDefaults() + +# Subdirectories inside the HF repo (see config.json::checkpoint_directories). +_IMAGE_CKPT_DIR = "Lance_3B" +_VIDEO_CKPT_DIR = "Lance_3B_Video" +_VIT_DIR = "Qwen2.5-VL-ViT" +_VAE_FILE = "Wan2.2_VAE.pth" + + +def get_lance_post_process_func(od_config: OmniDiffusionConfig): + """Lance returns PIL.Image.Image directly, same as BAGEL.""" + + def post_process_func(x): + return x + + return post_process_func + + +def get_lance_pre_process_func(od_config: OmniDiffusionConfig): + def pre_process_func(x): + return x + + return pre_process_func + + +@dataclass +class _LanceVaeCfg: + z_channels: int = LANCE_DEFAULTS.vae_z_channels + # BAGEL core computes ``latent_downsample = vae.downsample * latent_patch_size``. + # For Lance image latents that is ``16 * 2 = 32``. + downsample: int = LANCE_DEFAULTS.vae_downsample_spatial + + +@dataclass +class _LanceVitCfg: + patch_size: int = 14 + hidden_size: int = 1280 # Qwen2.5-VL vision hidden (out_hidden_size=2048) + + +class LancePipeline(BagelPipeline): + """Lance pipeline. Inherits BAGEL's forward/generation; overrides only + construction (checkpoint layout, Qwen2.5-VL ViT, Wan2.2 VAE).""" + + def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): + # Intentionally do NOT call BagelPipeline.__init__ — its assumptions + # about config.json / vit_config.json / SigLIP / the BAGEL AE do not + # hold for Lance. We replicate the BAGEL construction sequence with + # Lance-specific component builders, then reuse every inherited method. + import torch.nn as nn + from vllm.transformers_utils.configs.bagel import BagelConfig + + from vllm_omni.diffusion.distributed.utils import get_local_device + from vllm_omni.diffusion.model_loader.diffusers_loader import ( + DiffusersPipelineLoader, + ) + + nn.Module.__init__(self) + self.od_config = od_config + self.device = get_local_device() + self.scheduler = None + self.scheduler_kwargs = {} + + model = od_config.model + if os.path.exists(model): + repo_root = model + else: + repo_root = download_weights_from_hf_specific(model, od_config.revision, ["*"]) + + # If --model points directly at ``Lance_3B`` / ``Lance_3B_Video`` (or + # ``Qwen2.5-VL-ViT``), walk up so ``repo_root`` is the bundled top-level + # dir that owns the sibling components (VAE, ViT). + base = os.path.basename(repo_root.rstrip("/")) + if base in {_IMAGE_CKPT_DIR, _VIDEO_CKPT_DIR, _VIT_DIR}: + parent = os.path.dirname(repo_root.rstrip("/")) + if os.path.isdir(parent) and os.path.isfile(os.path.join(parent, _VAE_FILE)): + repo_root = parent + + is_video = self._select_video_variant(od_config) or base == _VIDEO_CKPT_DIR + ckpt_dir = _VIDEO_CKPT_DIR if is_video else _IMAGE_CKPT_DIR + ckpt_path = os.path.join(repo_root, ckpt_dir) + if not os.path.isdir(ckpt_path): + # Some users point --model directly at a single checkpoint dir. + ckpt_path = repo_root + self._repo_root = repo_root + self._ckpt_path = ckpt_path + self._is_video = is_video + if is_video: + logger.info( + "Lance video checkpoint selected (%s) — wired with " + "LancePositionEmbedding3D and Wan2.2 multi-frame VAE decode " + "(text-to-video). Image-edit / x2t_video remain follow-ups.", + ckpt_dir, + ) + + # ---- LLM (Qwen2-MoT; identical weight layout to BAGEL) ---- + llm_cfg_path = os.path.join(ckpt_path, "llm_config.json") + llm_config = Qwen2MoTConfig.from_json_file(llm_cfg_path) + llm_config.qk_norm = True + # The released Lance checkpoint ships a separate ``language_model.lm_head.weight`` + # tensor even though ``llm_config.json`` says ``tie_word_embeddings=True``; keep + # the head untied so the checkpoint loads with zero missing/unexpected keys. + llm_config.tie_word_embeddings = False + llm_config.layer_module = od_config.override_transformer_cls_name or "Qwen2MoTDecoderLayer" + # Lance is Qwen2.5-VL-MoT and ships ``rope_scaling = {"type": "mrope", + # "mrope_section": [16, 24, 24]}``. Keep the mRoPE configuration: + # :class:`BagelRotaryEmbedding` now auto-dispatches on ``rope_type`` and + # consumes 3-D ``(t, h, w)`` position ids, while :class:`LanceBagel` + # broadcasts scalar positions to ``(3, S)`` for text-only blocks and + # emits true per-token ``(t, h, w)`` for video latent blocks. Required + # for the Qwen2.5-VL backbone to produce coherent x2t / t2v output. + if not isinstance(getattr(llm_config, "rope_scaling", None), dict): + llm_config.rope_scaling = {"rope_type": "mrope", "mrope_section": [16, 24, 24]} + else: + llm_config.rope_scaling.setdefault("rope_type", llm_config.rope_scaling.get("type", "mrope")) + llm_config.rope_scaling.setdefault("mrope_section", [16, 24, 24]) + + self.tokenizer = self._load_tokenizer(ckpt_path) + self.tokenizer, self.new_token_ids, _ = add_special_tokens(self.tokenizer) + # Image / video preprocessor for the understanding paths + # (x2t_image / x2t_video / image_edit / video_edit). Lance's HF repo + # does not bundle a ``preprocessor_config.json``; we use transformers' + # ``Qwen2VLImageProcessor`` / ``Qwen2VLVideoProcessor`` with their + # default CLIP-style normalization, ``patch_size=14``, ``merge_size=2``, + # which matches the bundled ``Qwen2.5-VL-ViT/config.json`` exactly. + self.image_processor = self._build_image_processor() + self.video_processor = self._build_video_processor() + tok_len = len(self.tokenizer) + required_max_id = max(int(v) for v in self.new_token_ids.values()) + llm_config.vocab_size = max( + int(getattr(llm_config, "vocab_size", tok_len)), + int(tok_len), + int(required_max_id + 1), + ) + + parallel_config = od_config.parallel_config if od_config else None + quant_config = od_config.quantization_config + + self.language_model = Qwen2MoTForCausalLM( + llm_config, + parallel_config=parallel_config, + quant_config=quant_config, + prefix="bagel.language_model", + ) + self.transformer = self.language_model.model + + # ---- Understanding ViT: Qwen2.5-VL vision (bundled) ---- + self.vit_model = self._build_qwen2_5_vl_vit(repo_root) + + # ---- VAE: Wan2.2 (bundled .pth) ---- + self.vae = self._build_wan22_vae(repo_root) + + vae_cfg = _LanceVaeCfg() + vit_cfg = _LanceVitCfg( + patch_size=14, + hidden_size=getattr(getattr(self.vit_model, "config", None), "hidden_size", 1280), + ) + + # Lance uses Qwen2.5-VL's vision tower whose ``merger`` already projects + # to the LLM hidden size and which carries its own positional encoding; + # there is no separate BAGEL-style ``connector`` / ``vit_pos_embed`` in + # the released checkpoint (Lance_3B safetensors only contain + # ``vit_model.*``, never ``connector.*`` / ``vit_pos_embed.*``). We keep + # BAGEL's ``visual_und=True`` so ``vit_model`` is registered as a child + # of ``self.bagel`` and BAGEL's ``forward_cache_update_vit`` works + # unchanged, then immediately replace ``connector`` with an identity and + # ``vit_pos_embed`` with a zero op so the strict load check does not + # demand those phantom weights and the addition in + # ``forward_cache_update_vit`` is numerically a no-op. + und_enabled = self.vit_model is not None + self.bagel = LanceBagel( + language_model=self.language_model, + vit_model=self.vit_model, + parallel_config=parallel_config, + quant_config=quant_config, + prefix="bagel", + config=BagelConfig( + llm_config=llm_config, + vae_config=vae_cfg, + vit_config=vit_cfg, + vit_max_num_patch_per_side=LANCE_DEFAULTS.vit_max_num_patch_per_side, + connector_act=LANCE_DEFAULTS.connector_act, + interpolate_pos=False, + latent_patch_size=LANCE_DEFAULTS.latent_patch_size_spatial, + max_latent_size=LANCE_DEFAULTS.max_latent_size, + timestep_shift=LANCE_DEFAULTS.timestep_shift, + visual_gen=True, + visual_und=und_enabled, + ), + ) + if und_enabled: + self.bagel.connector = LanceIdentityConnector() + self.bagel.vit_pos_embed = LanceZeroVitPosEmbed() + # Hand the Qwen2-VL processors to LanceBagel.prepare_vit_images / + # prepare_vit_videos so they can re-call them and recover + # ``image_grid_thw`` / ``video_grid_thw`` (BAGEL's lambda drops the + # grid). + self.bagel._lance_image_processor = self.image_processor + self.bagel._lance_video_processor = self.video_processor + + # Lance_3B_Video ships a 3-D latent positional table + # (``latent_pos_embed.pos_embed`` shape ``(max_num_frames * max_latent_size**2, hidden_size)``) + # in place of BAGEL's 2-D ``PositionEmbedding(max_latent_size, hidden)``; + # swap in :class:`LancePositionEmbedding3D` so the video checkpoint loads + # cleanly. ``max_num_frames`` is derived from the table size. + if is_video: + self.bagel.latent_pos_embed = LancePositionEmbedding3D( + max_num_frames=LANCE_DEFAULTS.max_num_video_latent_frames, + max_num_patch_per_side=LANCE_DEFAULTS.max_latent_size, + hidden_size=llm_config.hidden_size, + ) + + # Weight sources. Verified against bytedance-research/Lance: + # * Lance_3B/model.safetensors carries the LLM + connectors with + # keys ``language_model.* / vae2llm.* / llm2vae.* / time_embedder.* / + # latent_pos_embed.*`` — load under the ``bagel.`` namespace. + # * Lance_3B_Video/model.safetensors additionally includes + # ``vit_model.*`` (390 tensors). + # * Qwen2.5-VL-ViT/vit.safetensors (image checkpoint only) carries + # the bare ``blocks.* / merger.* / patch_embed.*`` and must land + # under ``bagel.vit_model.vision_model.*`` to match the + # :class:`LanceQwen2_5_VLNaViTWrapper` hierarchy. + # Always use the resolved repo_root for weight sources so subfolder paths + # resolve correctly even when the user passed a per-checkpoint subdir + # (we already walked ``repo_root`` up to the bundled root above). + weights_model = repo_root if os.path.isdir(repo_root) else od_config.model + self.weights_sources = [ + DiffusersPipelineLoader.ComponentSource( + model_or_path=weights_model, + subfolder=ckpt_dir if ckpt_path != repo_root else None, + revision=od_config.revision, + prefix="bagel.", + fall_back_to_pt=False, + ), + ] + # Always overlay ``Qwen2.5-VL-ViT/vit.safetensors`` when the bundle + # contains it. Upstream Lance's ``inference_lance.py`` loads ViT + # weights from ``vit.safetensors`` unconditionally (line 455-458), + # using the released understanding-ViT checkpoint regardless of t2i + # vs t2v / image_edit vs video_edit. Lance_3B_Video bundles its own + # ``vit_model.*`` inside ``model.safetensors``, but loading those + # produced byte-different ViT outputs from upstream (rel_l2 ≈ 9 + # at the merger output) — the bundled video-checkpoint ViT diverges + # from the standalone vit.safetensors. Mirroring upstream's load + # order (vit.safetensors LAST so it wins) restores byte-identical + # ViT outputs for both image_edit and video_edit. + if und_enabled and os.path.isdir(os.path.join(repo_root, _VIT_DIR)): + self.weights_sources.append( + DiffusersPipelineLoader.ComponentSource( + model_or_path=weights_model, + subfolder=_VIT_DIR, + revision=od_config.revision, + prefix="bagel.vit_model.vision_model.", + fall_back_to_pt=False, + ) + ) + + if quant_config is None and not (od_config.enable_layerwise_offload or od_config.parallel_config.use_hsdp): + self.to(self.device) + self.setup_diffusion_pipeline_profiler( + enable_diffusion_pipeline_profiler=self.od_config.enable_diffusion_pipeline_profiler + ) + + # ------------------------------------------------------------------ # + # Lance-specific component builders + # ------------------------------------------------------------------ # + @staticmethod + def _select_video_variant(od_config: OmniDiffusionConfig) -> bool: + """Pick Lance_3B vs Lance_3B_Video. Defaults to image; a user can + force video via the model path ending in the video dir or an + od_config extra flag.""" + model = str(od_config.model) + if model.rstrip("/").endswith(_VIDEO_CKPT_DIR): + return True + extra = getattr(od_config, "extra", None) or {} + return bool(extra.get("lance_video", False)) + + @staticmethod + def _load_tokenizer(ckpt_path: str): + from transformers import AutoTokenizer + + return AutoTokenizer.from_pretrained(ckpt_path, local_files_only=True, trust_remote_code=True) + + @staticmethod + def _build_image_processor(): + """Construct a Qwen2.5-VL-compatible image preprocessor. + + Defaults mirror ``Qwen/Qwen2.5-VL-3B-Instruct`` (CLIP normalization, + 14-pixel patches, 2x spatial merge). + """ + from transformers import Qwen2VLImageProcessor + + return Qwen2VLImageProcessor() + + @staticmethod + def _build_video_processor(): + """Construct a Qwen2.5-VL-compatible video preprocessor. + + Used by the ``x2t_video`` understanding path. Returns + ``pixel_values_videos`` + 3-D ``video_grid_thw = [T_lat, H, W]``. + """ + from transformers import Qwen2VLVideoProcessor + + return Qwen2VLVideoProcessor() + + def _build_qwen2_5_vl_vit(self, repo_root: str): + """Build the bundled Qwen2.5-VL vision tower and wrap it NaViT-style. + + Lance's bundled ``Qwen2.5-VL-ViT/config.json`` sets + ``_attn_implementation = "flash_attention_2"``; we force ``sdpa`` + instead so the tower constructs on hardware (e.g. Blackwell) where + ``flash-attn`` is not present, falling back to PyTorch's native + scaled-dot-product attention. Numerically equivalent for inference. + """ + vit_dir = os.path.join(repo_root, _VIT_DIR) + cfg_path = os.path.join(vit_dir, "config.json") + with open(cfg_path, encoding="utf-8") as f: + vit_cfg_dict = json.load(f) + vit_cfg_dict = dict(vit_cfg_dict) + vit_cfg_dict["_attn_implementation"] = "sdpa" + try: + from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import ( + Qwen2_5_VLVisionConfig, + ) + from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import ( + Qwen2_5_VisionTransformerPretrainedModel, + ) + + vit_conf = Qwen2_5_VLVisionConfig(**vit_cfg_dict) + vision = Qwen2_5_VisionTransformerPretrainedModel(vit_conf) + except Exception as e: + logger.warning( + "Could not instantiate Qwen2.5-VL vision tower (%s). " + "Understanding (x2t) path will be unavailable; generation (t2i) " + "does not use the ViT.", + e, + ) + return None + return LanceQwen2_5_VLNaViTWrapper(vision, spatial_merge_size=int(vit_cfg_dict.get("spatial_merge_size", 2))) + + # ------------------------------------------------------------------ # + # Shared prefill helpers used by image_edit / i2v / video_edit. + # Each mutates ``ctx`` in place: ``ctx["past_key_values"]`` / + # ``ctx["kv_lens"]`` / ``ctx["ropes"]``. + # ------------------------------------------------------------------ # + def _autocast_kwargs(self) -> dict: + return dict( + device_type=self.device.type, + enabled=self.device.type != "cpu", + dtype=self.od_config.dtype, + ) + + def _new_gen_context(self) -> dict: + from .lance_transformer import NaiveCache + + return { + "kv_lens": [0], + "ropes": [0], + "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), + } + + @staticmethod + def _segment_strings(task: str, modality: str) -> tuple[str, str]: + sys_prompt = SYSTEM_PROMPTS[(task, modality)] + return ( + f"<|im_start|>system\n{sys_prompt}<|im_end|>\n<|im_start|>user\n", + "<|im_end|>\n<|im_start|>assistant\n", + ) + + def _raw_text_prefill(self, ctx: dict, text_str: str) -> None: + """Tokenize ``text_str`` (no bos/eos) and append to ``ctx`` KV cache.""" + text_ids = self.tokenizer.encode(text_str, add_special_tokens=False) + if not text_ids: + return + curr_kvlen = ctx["kv_lens"][0] + curr_rope = ctx["ropes"][0] + seq_len = len(text_ids) + inp = { + "text_token_lens": torch.tensor([seq_len], dtype=torch.int, device=self.device), + "packed_text_ids": torch.tensor(text_ids, dtype=torch.long, device=self.device), + "packed_text_position_ids": torch.arange( + curr_rope, curr_rope + seq_len, dtype=torch.long, device=self.device + ), + } + with torch.autocast(**self._autocast_kwargs()): + ctx["past_key_values"] = self.bagel.forward_cache_update_text(ctx["past_key_values"], **inp) + ctx["kv_lens"] = [curr_kvlen + seq_len] + ctx["ropes"] = [curr_rope + seq_len] + + # ``prepare_vit_*`` returns ``packed_indexes`` / ``packed_key_value_indexes`` + # / ``key_values_lens`` for historical reasons; ``forward_cache_update_vit`` + # post-main-merge no longer accepts them. + _STALE_PREFILL_KEYS = ("packed_indexes", "packed_key_value_indexes", "key_values_lens") + + def _vit_image_prefill(self, ctx: dict, images: list, transforms) -> None: + """ViT prefill from PIL images via ``prepare_vit_images``.""" + inp, new_kvlens, new_rope = self.bagel.prepare_vit_images( + curr_kvlens=ctx["kv_lens"], + curr_rope=ctx["ropes"], + images=images, + transforms=transforms, + new_token_ids=self.new_token_ids, + ) + for k, v in inp.items(): + if torch.is_tensor(v): + inp[k] = v.to(self.device) + for k in self._STALE_PREFILL_KEYS: + inp.pop(k, None) + with torch.autocast(**self._autocast_kwargs()): + ctx["past_key_values"] = self.bagel.forward_cache_update_vit(ctx["past_key_values"], **inp) + ctx["kv_lens"] = new_kvlens + ctx["ropes"] = new_rope + + def _vit_video_prefill( + self, + ctx: dict, + videos: list, + precomputed_vit: list | None = None, + ) -> None: + """ViT prefill from videos via ``prepare_vit_videos``. + + Pass ``precomputed_vit=[(pixels, grid_thw)]`` to bypass the + Qwen2VLImageProcessor smart-resize and feed already-bucketed + ViT patches directly (used by i2v / video_edit to keep grid + dimensions aligned with the ref VAE block). Omit it to let + ``prepare_vit_videos`` run the processor itself (x2t_video). + """ + kwargs = dict( + curr_kvlens=ctx["kv_lens"], + curr_rope=ctx["ropes"], + videos=videos, + new_token_ids=self.new_token_ids, + ) + if precomputed_vit is not None: + kwargs["precomputed_vit"] = precomputed_vit + inp, new_kvlens, new_rope = self.bagel.prepare_vit_videos(**kwargs) + for k, v in inp.items(): + if torch.is_tensor(v): + inp[k] = v.to(self.device) + for k in self._STALE_PREFILL_KEYS: + inp.pop(k, None) + with torch.autocast(**self._autocast_kwargs()): + ctx["past_key_values"] = self.bagel.forward_cache_update_vit(ctx["past_key_values"], **inp) + ctx["kv_lens"] = new_kvlens + ctx["ropes"] = new_rope + + def _vae_ref_prefill( + self, + ctx: dict, + images: list, + transforms, + *, + is_video: bool, + latent_cache: list, + ) -> None: + """VAE prefill of a reference image / video at timestep 0. + + ``latent_cache`` is a single-slot list used to share the encoded + posterior between the gen branch and the cfg_text branch — Wan2.2's + ``mu + std * randn_like(std)`` samples a fresh latent every call, + which otherwise drifts the two branches' KV caches. + """ + inp, new_kvlens, new_rope = self.bagel._lance_native_prepare_vae_images( + curr_kvlens=ctx["kv_lens"], + curr_rope=ctx["ropes"], + images=images, + transforms=transforms, + new_token_ids=self.new_token_ids, + is_video=is_video, + ) + for k, v in inp.items(): + if torch.is_tensor(v): + inp[k] = v.to(self.device) + if not latent_cache: + latent_cache.append(self.vae.encode(inp["padded_images"].to(self.device))) + inp["precomputed_latent"] = latent_cache[0] + with torch.autocast(**self._autocast_kwargs()): + ctx["past_key_values"] = self.bagel.forward_cache_update_vae(self.vae, ctx["past_key_values"], **inp) + ctx["kv_lens"] = new_kvlens + ctx["ropes"] = new_rope + + # ------------------------------------------------------------------ # + # Lance text-to-video forward path + # ------------------------------------------------------------------ # + def forward(self, req): # type: ignore[override] + """Dispatch on prompt modality. + + - ``modalities == ["video"]`` (text-to-video) → :meth:`_forward_t2v` + (3-D latents + ``LanceWanVAE.decode_video``). + - ``modalities == ["text"]`` + ``multi_modal_data.video`` (x2t_video) → + :meth:`_forward_x2t_video` (multi-frame Qwen2.5-VL ViT prefill). + - ``modalities == ["image"]`` + ``multi_modal_data.img2img`` (image_edit) + → :meth:`_forward_image_edit` (Lance-native VAE+ViT prefill + image gen). + - ``modalities == ["video"]`` + ``multi_modal_data.video`` (video_edit) + → :meth:`_forward_video_edit` (Lance-native multi-frame VAE+ViT prefill + + video gen). + - Everything else falls through to :meth:`BagelPipeline.forward` + (t2i, x2t_image). + """ + first_prompt = req.prompts[0] if req.prompts else None + modalities: list[str] = [] + mm_data: dict = {} + if isinstance(first_prompt, dict): + modalities = first_prompt.get("modalities") or [] + mm_data = first_prompt.get("multi_modal_data") or {} + if "video" in modalities and mm_data.get("first_frame") is not None: + return self._forward_i2v(req) + if "video" in modalities and mm_data.get("video") is not None: + return self._forward_video_edit(req) + if "video" in modalities: + return self._forward_t2v(req) + if "text" in modalities and mm_data.get("video") is not None: + return self._forward_x2t_video(req) + if "text" in modalities and mm_data.get("image") is not None: + return self._forward_x2t_image(req) + if "image" in modalities and (mm_data.get("img2img") is not None or mm_data.get("image") is not None): + return self._forward_image_edit(req) + # t2i falls through to BAGEL parent. Inject Lance defaults if the + # caller hasn't set them: upstream uses ``cfg_text_scale=4.0`` and + # ``cfg_vit_scale=1.0`` (= cfg_img_scale=1.0 in BAGEL terms). + sp = req.sampling_params + if not hasattr(sp, "extra_args") or sp.extra_args is None: + sp.extra_args = {} + sp.extra_args.setdefault("cfg_img_scale", 1.0) + return super().forward(req) + + @torch.inference_mode() + def _forward_t2v(self, req): + """Minimal text-to-video forward: text prefill + 3-D latent denoising + + Wan2.2 multi-frame decode. Mirrors :meth:`BagelPipeline.forward`'s t2i + branch but with 3-D latents (no image input, no x2t).""" + from copy import deepcopy + + from vllm_omni.diffusion.data import DiffusionOutput + + from .lance_transformer import NaiveCache + + first_prompt = req.prompts[0] + if isinstance(first_prompt, dict): + prompt = first_prompt.get("prompt") or "" + extra_args = first_prompt.get("extra_args") or {} + else: + prompt = str(first_prompt) + extra_args = {} + # Sampling-side extras override prompt-side. + sp_extra = getattr(req.sampling_params, "extra_args", {}) or {} + extra_args = {**extra_args, **sp_extra} + + # Video shape. T = number of RGB frames (1..121), H/W in pixels. + T = int(extra_args.get("num_frames", 25)) + H = int(req.sampling_params.height or extra_args.get("video_height", 480)) + W = int(req.sampling_params.width or extra_args.get("video_width", 768)) + max_lat = self.bagel.max_latent_size + max_hw = max_lat * self.bagel.latent_downsample + if H > max_hw or W > max_hw: + raise ValueError(f"Requested video resolution {H}x{W} exceeds Lance limit {max_hw}x{max_hw}") + downsample_t = int(getattr(self.bagel.config.vae_config, "downsample_temporal", 4)) + max_T_lat = LANCE_DEFAULTS.max_num_video_latent_frames + if (T - 1) // downsample_t + 1 > max_T_lat: + raise ValueError( + f"Requested num_frames={T} exceeds Lance video temporal limit " + f"{(max_T_lat - 1) * downsample_t + 1} (max_num_video_latent_frames={max_T_lat})" + ) + video_shape = (T, H, W) + logger.info("Lance t2v: video_shape=%s", video_shape) + + cfg_text_scale = float(extra_args.get("cfg_text_scale", 4.0)) + # Upstream Lance defaults cfg_interval=[0.4, 1.0] (turn off CFG below + # t=0.4); without this vllm-omni keeps CFG on for late iters and the + # last few denoise steps diverge from upstream. cfg_renorm_type / + # cfg_renorm_min default to 'global' / 0 on both sides. + cfg_interval = tuple(extra_args.get("cfg_interval", (0.4, 1.0))) + cfg_renorm_type = str(extra_args.get("cfg_renorm_type", "global")) + cfg_renorm_min = float(extra_args.get("cfg_renorm_min", 0.0)) + timestep_shift = float(extra_args.get("timestep_shift", LANCE_DEFAULTS.timestep_shift)) + num_timesteps = int(req.sampling_params.num_inference_steps or LANCE_DEFAULTS.num_timesteps) + + if req.sampling_params.seed is not None: + torch.manual_seed(req.sampling_params.seed) + if self.device.type == "cuda": + torch.cuda.manual_seed(req.sampling_params.seed) + + # ---- Build positive-prompt KV cache ---- + gen_context = { + "kv_lens": [0], + "ropes": [0], + "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), + } + cfg_text_context = deepcopy(gen_context) + + gen_input_text, newlens, new_rope = self.bagel.prepare_prompts( + curr_kvlens=gen_context["kv_lens"], + curr_rope=gen_context["ropes"], + prompts=[prompt], + tokenizer=self.tokenizer, + new_token_ids=self.new_token_ids, + ) + for k, v in gen_input_text.items(): + if torch.is_tensor(v): + gen_input_text[k] = v.to(self.device) + with torch.autocast( + device_type=self.device.type, + enabled=self.device.type != "cpu", + dtype=self.od_config.dtype, + ): + gen_context["past_key_values"] = self.bagel.forward_cache_update_text( + gen_context["past_key_values"], **gen_input_text + ) + gen_context["kv_lens"] = newlens + gen_context["ropes"] = new_rope + + # ---- Build CFG text-unconditional KV cache (empty prompt) ---- + if cfg_text_scale > 1.0: + neg_prompt = str(extra_args.get("negative_prompt") or "") + neg_input, neg_newlens, neg_rope = self.bagel.prepare_prompts( + curr_kvlens=cfg_text_context["kv_lens"], + curr_rope=cfg_text_context["ropes"], + prompts=[neg_prompt], + tokenizer=self.tokenizer, + new_token_ids=self.new_token_ids, + ) + for k, v in neg_input.items(): + if torch.is_tensor(v): + neg_input[k] = v.to(self.device) + with torch.autocast( + device_type=self.device.type, + enabled=self.device.type != "cpu", + dtype=self.od_config.dtype, + ): + cfg_text_context["past_key_values"] = self.bagel.forward_cache_update_text( + cfg_text_context["past_key_values"], **neg_input + ) + cfg_text_context["kv_lens"] = neg_newlens + cfg_text_context["ropes"] = neg_rope + + # ---- 3-D latent init + CFG side metadata ---- + gen_input_lat = self.bagel.prepare_video_latent( + curr_kvlens=gen_context["kv_lens"], + curr_rope=gen_context["ropes"], + video_shapes=[video_shape], + new_token_ids=self.new_token_ids, + ) + for k, v in gen_input_lat.items(): + if torch.is_tensor(v): + gen_input_lat[k] = v.to(self.device) + cfg_text_lat = self.bagel.prepare_video_latent_cfg( + curr_kvlens=cfg_text_context["kv_lens"], + curr_rope=cfg_text_context["ropes"], + video_shapes=[video_shape], + ) + cfg_img_lat = self.bagel.prepare_video_latent_cfg( + curr_kvlens=gen_context["kv_lens"], + curr_rope=gen_context["ropes"], + video_shapes=[video_shape], + ) + for k, v in cfg_text_lat.items(): + if torch.is_tensor(v): + cfg_text_lat[k] = v.to(self.device) + for k, v in cfg_img_lat.items(): + if torch.is_tensor(v): + cfg_img_lat[k] = v.to(self.device) + + self._regen_init_noise_on_device(gen_input_lat, req.sampling_params.seed) + + # ---- Denoising loop (Bagel.generate_image is rank-agnostic over packed tokens) ---- + with torch.autocast( + device_type=self.device.type, + enabled=self.device.type != "cpu", + dtype=self.od_config.dtype, + ): + # ``prepare_video_latent`` / ``prepare_vae_latent`` still return + # ``key_values_lens`` / ``packed_indexes`` / ``packed_key_value_indexes`` + # but post-main-merge ``generate_image`` doesn't accept them — drop + # before unpacking via ``**gen_input_lat``. + for _drop in ("key_values_lens", "packed_indexes", "packed_key_value_indexes"): + gen_input_lat.pop(_drop, None) + latents, *_ = self.bagel.generate_image( + past_key_values=gen_context["past_key_values"], + cfg_text_past_key_values=cfg_text_context["past_key_values"], + cfg_img_past_key_values=gen_context["past_key_values"], # no img CFG branch + num_timesteps=num_timesteps, + timestep_shift=timestep_shift, + cfg_text_scale=cfg_text_scale, + cfg_img_scale=1.0, + cfg_interval=cfg_interval, + cfg_renorm_type=cfg_renorm_type, + cfg_renorm_min=cfg_renorm_min, + **gen_input_lat, + cfg_text_packed_position_ids=cfg_text_lat["cfg_packed_position_ids"], + # ``cfg_text_packed_query_indexes`` / ``cfg_text_key_values_lens`` + # / ``cfg_text_packed_key_value_indexes`` removed post-main-merge + # (derived from ``cfg_text_past_key_values``). + cfg_img_packed_position_ids=cfg_img_lat["cfg_packed_position_ids"], + # ``cfg_img_*`` index/lens kwargs removed — same as above. + ) + + frames_np = self._decode_video_from_latent(self.bagel, self.vae, latents[0], video_shape) + # Convert numpy frames to PIL.Image list for downstream serialization. + frames = [Image.fromarray(f) for f in frames_np] + logger.info("Lance t2v: decoded %d frames at %dx%d", len(frames), frames[0].width, frames[0].height) + return DiffusionOutput( + output=frames[0], + custom_output={"video_frames": frames, "video_shape": video_shape}, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ) + + @torch.inference_mode() + def _forward_image_edit(self, req): + """Image edit (img2img): reference image + text prompt → modified image. + + Matches upstream Lance's image_edit prefill structure (see + ``Lance.validation_gen_KVcache`` and ``validation_dataset.py``). The + sequence is laid out as 5 prefill segments + 1 noise QUERY: + + seg1 (causal, modality=system_prompt): + ``<|im_start|>system\\n{sys}<|im_end|>\\n<|im_start|>user\\n`` + seg2 (full, modality=ref_vit): + ViT(ref) — ref image through Qwen2.5-VL ViT + seg3 (full_noise, modality=ref_source): + VAE(ref) — ref image through Wan2.2 VAE, time_embed(t=0) + seg4 (causal, modality=text): the user instruction + seg5 (causal, modality=system_prompt): + ``<|im_end|>\\n<|im_start|>assistant\\n`` + seg6 (noise QUERY): the gen latent to denoise + + cfg_text_context skips ONLY seg4 (the user text instruction) — every + other segment, including the system header, ViT and VAE prefill, the + separator and the noise query, is shared between gen_context and + cfg_text_context. The rope counter still advances by seg4's length + for cfg_text_context so that seg5's rope position matches between + the two branches. + + cfg_img is OFF by default (``cfg_img_scale=1.0`` matches upstream's + ``cfg_vit_scale=1.0``). + """ + + import numpy as np + + from vllm_omni.diffusion.data import DiffusionOutput + + from .lance_transformer import NaiveCache + + first_prompt = req.prompts[0] + assert isinstance(first_prompt, dict), "image_edit requires dict-style prompt" + # The caller may pass either a raw user instruction (e.g. "Remove the + # hat from the painting.") OR an already-rendered template string + # (with system + chat template). Extract just the user instruction + # so we can rebuild the segments cleanly. + user_text = first_prompt.get("user_text") + rendered = first_prompt.get("prompt") or "" + if not user_text: + user_text = _extract_user_instruction(rendered) + mm_data = first_prompt.get("multi_modal_data") or {} + image_input = mm_data.get("img2img") or mm_data.get("image") + if image_input is None: + raise ValueError("image_edit requires multi_modal_data.img2img (or .image).") + if not isinstance(image_input, list): + image_input = [image_input] + image_input = [Image.open(im) if isinstance(im, str) else im for im in image_input] + + extra_args = first_prompt.get("extra_args") or {} + sp_extra = getattr(req.sampling_params, "extra_args", {}) or {} + extra_args = {**extra_args, **sp_extra} + cfg_text_scale = float(extra_args.get("cfg_text_scale", 4.0)) + cfg_img_scale = float(extra_args.get("cfg_img_scale", 1.0)) + cfg_interval = tuple(extra_args.get("cfg_interval", (0.4, 1.0))) + cfg_renorm_type = str(extra_args.get("cfg_renorm_type", "global")) + cfg_renorm_min = float(extra_args.get("cfg_renorm_min", 0.0)) + timestep_shift = float(extra_args.get("timestep_shift", LANCE_DEFAULTS.timestep_shift)) + num_timesteps = int(req.sampling_params.num_inference_steps or LANCE_DEFAULTS.num_timesteps) + + # Resize the reference image to a multiple of latent_downsample and + # within the max latent grid. + stride = self.bagel.latent_downsample + max_hw = int(self.bagel.max_latent_size * stride) + + def _resize_to_stride(img): + if img.mode != "RGB": + img = img.convert("RGB") + w, h = img.size + scale = min(max_hw / max(w, h), 1.0) + scale = max(scale, min(256, max_hw) / min(w, h)) + new_w = max(stride, int(round(w * scale / stride) * stride)) + new_h = max(stride, int(round(h * scale / stride) * stride)) + new_w = min(new_w, max_hw) + new_h = min(new_h, max_hw) + if new_w != w or new_h != h: + img = img.resize((new_w, new_h), Image.BICUBIC) + return img + + image_input = [_resize_to_stride(im) for im in image_input] + resized_w, resized_h = image_input[0].size + image_shape = (resized_h, resized_w) + logger.info("Lance image_edit: ref image %dx%d, user_text=%r", resized_w, resized_h, user_text) + + def vae_transforms(img): + arr = torch.from_numpy(np.array(img)).float() / 127.5 - 1.0 + return arr.permute(2, 0, 1) # (C, H, W) + + def vit_transforms(img): + return self.image_processor(images=img, return_tensors="pt").pixel_values[0] + + if req.sampling_params.seed is not None: + torch.manual_seed(req.sampling_params.seed) + if self.device.type == "cuda": + torch.cuda.manual_seed(req.sampling_params.seed) + + gen_context = { + "kv_lens": [0], + "ropes": [0], + "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), + } + cfg_text_context = { + "kv_lens": [0], + "ropes": [0], + "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), + } + + # Single-slot cache shared between gen + cfg branches so the Wan2.2 + # VAE's ``mu + std * randn_like(std)`` sample is computed once. + ref_latent_cache: list = [] + + # ----- prefill segments in upstream order ----- + seg1_str, seg5_str = self._segment_strings("image_edit", "image") + + # seg1: system + user header (shared) + self._raw_text_prefill(gen_context, seg1_str) + if cfg_text_scale > 1.0: + self._raw_text_prefill(cfg_text_context, seg1_str) + + # seg2: ViT(ref) (shared) + self._vit_image_prefill(gen_context, image_input, vit_transforms) + if cfg_text_scale > 1.0: + self._vit_image_prefill(cfg_text_context, image_input, vit_transforms) + + # seg3: VAE(ref) with timestep=0 (shared) + # + # IMPORTANT: upstream Lance places the gen latent (the noise QUERY) + # at the SAME mRoPE positions as the ref VAE — they share the same + # spatial grid (T_lat × H_lat × W_lat), and the model treats them + # as two timesteps of the same latent location. Concretely, upstream + # ``get_rope_index`` emits ``[base_t, base_h+hi, base_w+wi]`` for + # both the VAE cond block AND the noise block, anchored at the SAME + # base. This is the inductive bias that lets the model "edit" the + # ref image — without this overlap the model has no way to map noise + # tokens to ref tokens. + # + # We therefore snapshot ``ropes`` BEFORE the VAE prefill and reuse + # that as the gen latent's ``curr_rope`` below. + rope_before_vae = gen_context["ropes"][0] + cfg_rope_before_vae = cfg_text_context["ropes"][0] if cfg_text_scale > 1.0 else rope_before_vae + self._vae_ref_prefill(gen_context, image_input, vae_transforms, is_video=False, latent_cache=ref_latent_cache) + if cfg_text_scale > 1.0: + self._vae_ref_prefill( + cfg_text_context, image_input, vae_transforms, is_video=False, latent_cache=ref_latent_cache + ) + + # seg4: user instruction (gen only). Upstream Lance's cfg_text + # branch is a SHORTER sequence with the user_text segment REMOVED + # (see ``uncond_split_pro_kvcache`` + ``get_rope_index`` on the + # filtered text_ids), so cfg_text's rope counter MUST NOT advance + # past the user_text region. seg5 (separator) therefore lands at + # different absolute rope positions in gen vs cfg branches — that's + # the intended behavior. + self._raw_text_prefill(gen_context, user_text) + + # seg5: separator + assistant header (gen + cfg). Each branch + # places it at its OWN current rope. + self._raw_text_prefill(gen_context, seg5_str) + if cfg_text_scale > 1.0: + self._raw_text_prefill(cfg_text_context, seg5_str) + + # -- (4) Gen latent at the SAME rope position as the ref VAE block. + # See the comment on `rope_before_vae` above. Note the KV cache + # still contains all prefilled segments (system, ViT, ref VAE, user + # text, separator); only the gen latent's QUERY rope coordinates are + # rewound to overlap with the ref VAE. + gen_input_lat = self.bagel.prepare_vae_latent( + curr_kvlens=gen_context["kv_lens"], + curr_rope=[rope_before_vae], + image_sizes=[image_shape], + new_token_ids=self.new_token_ids, + ) + for k, v in gen_input_lat.items(): + if torch.is_tensor(v): + gen_input_lat[k] = v.to(self.device) + cfg_text_lat = self.bagel.prepare_vae_latent_cfg( + curr_kvlens=cfg_text_context["kv_lens"], + curr_rope=[cfg_rope_before_vae], + image_sizes=[image_shape], + ) + for k, v in cfg_text_lat.items(): + if torch.is_tensor(v): + cfg_text_lat[k] = v.to(self.device) + + # cfg_img branch: keep off (cfg_img_scale=1.0 matches upstream + # ``cfg_vit_scale=1.0``). generate_image needs the cfg_img_* args + # but skips the branch when scale<=1.0; just point them at the + # cond's lat metadata so shapes/types are fine. + cfg_img_lat = cfg_text_lat + + self._regen_init_noise_on_device(gen_input_lat, req.sampling_params.seed) + + with torch.autocast(**self._autocast_kwargs()): + # ``prepare_video_latent`` / ``prepare_vae_latent`` still return + # ``key_values_lens`` / ``packed_indexes`` / ``packed_key_value_indexes`` + # but post-main-merge ``generate_image`` doesn't accept them — drop + # before unpacking via ``**gen_input_lat``. + for _drop in ("key_values_lens", "packed_indexes", "packed_key_value_indexes"): + gen_input_lat.pop(_drop, None) + latents, *_ = self.bagel.generate_image( + past_key_values=gen_context["past_key_values"], + cfg_text_past_key_values=cfg_text_context["past_key_values"] + if cfg_text_scale > 1.0 + else gen_context["past_key_values"], + cfg_img_past_key_values=gen_context["past_key_values"], + num_timesteps=num_timesteps, + timestep_shift=timestep_shift, + cfg_text_scale=cfg_text_scale, + cfg_img_scale=cfg_img_scale, + cfg_interval=cfg_interval, + cfg_renorm_type=cfg_renorm_type, + cfg_renorm_min=cfg_renorm_min, + **gen_input_lat, + cfg_text_packed_position_ids=cfg_text_lat["cfg_packed_position_ids"], + # ``cfg_text_packed_query_indexes`` / ``cfg_text_key_values_lens`` + # / ``cfg_text_packed_key_value_indexes`` removed post-main-merge + # (derived from ``cfg_text_past_key_values``). + cfg_img_packed_position_ids=cfg_img_lat["cfg_packed_position_ids"], + # ``cfg_img_*`` index/lens kwargs removed — same as above. + ) + + img = self._decode_image_from_latent(self.bagel, self.vae, latents[0], image_shape) + # Put the PIL image into ``custom_output`` too — the orchestrator + # serializes ``custom_output`` across the IPC boundary but strips + # the bare ``output`` field, so consumers that only see + # ``outputs[0].output`` (e.g. gradio_demo) get ``None`` otherwise. + return DiffusionOutput( + output=img, + custom_output={"image": img, "image_shape": image_shape}, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ) + + @torch.inference_mode() + @torch.inference_mode() + def _forward_i2v(self, req): + """Image-to-Video (upstream-faithful first-frame conditioning). + + Mirrors upstream Lance PR #33's ``ff2v_sample`` + ``validation_gen_KVcache`` + math byte-for-byte at the algorithmic level: + + 1. Treat the input image as 4 replicated pixel frames + ``(3, 4, H, W)``. This matches upstream + ``vae_tensor[:, :4, :, :] = vae_image_tensor[:, 0:1, :, :].repeat(1, 4, 1, 1)`` + — required because Wan2.2 VAE has temporal stride 4, so 4 + pixel frames collapse to 1 latent frame. + + 2. VAE-encode → first latent frame at the image content. + + 3. Inject the encoded latent into ``packed_init_noises`` at + the first ``h_lat × w_lat`` token slice (frame 0). + + 4. Pass ``frame_condition_token_indexes = arange(0, h_lat*w_lat)`` + to ``Bagel.generate_image`` so that: + * ``timestep[cond] = 0`` every iter (clean signal), + * ``x_t[cond]`` is restored verbatim after each + velocity update (never denoised). + + Plus a ViT+VAE-ref prefill of the same image (so the LLM also + attends to it through KV cache, matching upstream's + gen-pre-condition pass). + """ + + import numpy as _np + from PIL import Image as _PILImage + + from vllm_omni.diffusion.data import DiffusionOutput + + from .lance_transformer import NaiveCache + + first_prompt = req.prompts[0] + assert isinstance(first_prompt, dict), "i2v requires dict-style prompt" + user_text = first_prompt.get("user_text") + rendered = first_prompt.get("prompt") or "" + if not user_text: + user_text = _extract_user_instruction(rendered) + mm_data = first_prompt.get("multi_modal_data") or {} + image_input = mm_data.get("first_frame") + if image_input is None: + raise ValueError("i2v requires multi_modal_data.first_frame.") + + # Resolve image to (H, W, 3) uint8 RGB, then wrap as 1-frame video. + if isinstance(image_input, _PILImage.Image): + image_raw = _np.array(image_input.convert("RGB")) + elif isinstance(image_input, str): + image_raw = _np.array(_PILImage.open(image_input).convert("RGB")) + elif isinstance(image_input, _np.ndarray): + image_raw = image_input if image_input.ndim == 3 else image_input.squeeze() + else: + raise ValueError(f"Unsupported first_frame type {type(image_input)}") + video_raw = image_raw[None] # (1, H, W, 3) + + extra_args = first_prompt.get("extra_args") or {} + sp_extra = getattr(req.sampling_params, "extra_args", {}) or {} + extra_args = {**extra_args, **sp_extra} + + # Output video shape — independent of input image dims. + num_frames_out = int(extra_args.get("num_frames", 61)) + out_H = int(extra_args.get("video_height", 480)) + out_W = int(extra_args.get("video_width", 848)) + origin_fps = float(extra_args.get("origin_fps", 24.0)) + + # Upstream Lance i2v default is cfg_text_scale=4.0. Note: vllm-omni's + # bf16 attention trajectory (vllm-native kernels vs upstream's + # HF-native ones) compounds per-layer drift that may DAMPEN prompt- + # directed motion for subtle prompts (e.g. micro facial expression). + # For such cases, boost cfg_text_scale to 10-15 via + # ``extra_args["cfg_text_scale"]`` to amplify prompt influence. + # WARNING: too-high cfg (e.g. 15) can cause anatomical distortion + # for natural-motion prompts (e.g. animals walking). Tune per case. + cfg_text_scale = float(extra_args.get("cfg_text_scale", 4.0)) + cfg_img_scale = float(extra_args.get("cfg_img_scale", 1.0)) + # Upstream Lance i2v default (config/config_factory.py:cfg_interval): + # [0.4, 1.0] — CFG active in high-noise phase only. Always-on CFG + # ``(0, 1)`` over-constrains the late-step velocity toward the + # conditioned first frame, suppressing subtle motion (e.g. micro + # facial expression changes in ex5). + cfg_interval = tuple(extra_args.get("cfg_interval", (0.4, 1.0))) + cfg_renorm_type = str(extra_args.get("cfg_renorm_type", "global")) + cfg_renorm_min = float(extra_args.get("cfg_renorm_min", 0.0)) + timestep_shift = float(extra_args.get("timestep_shift", LANCE_DEFAULTS.timestep_shift)) + num_timesteps = int(req.sampling_params.num_inference_steps or LANCE_DEFAULTS.num_timesteps) + + # Bucket-resize + VAE preprocess the 1-frame "video" (image). i2v + # does not run a ViT prefill on the ref (the first-frame latent pin + # provides all the image conditioning), so the ViT outputs are + # discarded — see the comment further down on upstream PR #33. + from .lance_transformer import LanceBagel as _LanceBagel + + vae_video, _, _, _ = _LanceBagel._lance_video_preprocess(video_raw, origin_fps) + T_ref = int(vae_video.shape[1]) + H_ref = int(vae_video.shape[2]) + W_ref = int(vae_video.shape[3]) + + # Target output shape (independent of ref) — pass through the same + # VAE-aligned bucket so the gen latent dims are valid. We pick H/W + # from extra_args and snap to the VAE downsample grid (×8 spatial, + # ×4 temporal for Wan2.2 VAE). num_frames_out controls T. + # Wan2.2 latent dims: t_lat = (T - 1) // 4 + 1, h_lat = H // 8, w_lat = W // 8. + out_shape = (num_frames_out, out_H, out_W) + + logger.info( + "Lance i2v: ref image %dx%d → 1-frame VAE %dx%dx%d, target output %dx%dx%d (T,H,W), user_text=%r", + image_raw.shape[0], + image_raw.shape[1], + T_ref, + H_ref, + W_ref, + num_frames_out, + out_H, + out_W, + user_text, + ) + + if req.sampling_params.seed is not None: + torch.manual_seed(req.sampling_params.seed) + if self.device.type == "cuda": + torch.cuda.manual_seed(req.sampling_params.seed) + + gen_context = { + "kv_lens": [0], + "ropes": [0], + "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), + } + cfg_text_context = { + "kv_lens": [0], + "ropes": [0], + "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), + } + + # i2v system prompt (matches t2v wording per upstream). + seg1_str, seg5_str = self._segment_strings("i2v", "video") + + # Upstream Lance PR #33 i2v does NOT prefill the input image via + # ViT/VAE as a "reference". The image conditioning happens entirely + # through the first-frame latent pin (cond positions in x_t set to + # VAE-encoded image, never updated through denoise loop). Prefill + # is text-only: system prompt → user instruction → assistant header. + # Adding ViT/VAE prefill (which video_edit does) over-constrains the + # generation and prevents the prompt-driven motion from developing. + self._raw_text_prefill(gen_context, seg1_str) + if cfg_text_scale > 1.0: + self._raw_text_prefill(cfg_text_context, seg1_str) + + # Snapshot rope BEFORE adding user text; gen latent will use this + # as its starting rope (matches upstream's modality==1≡2 trick). + rope_before_vae = gen_context["ropes"][0] + cfg_rope_before_vae = cfg_text_context["ropes"][0] if cfg_text_scale > 1.0 else rope_before_vae + + self._raw_text_prefill(gen_context, user_text) + self._raw_text_prefill(gen_context, seg5_str) + if cfg_text_scale > 1.0: + self._raw_text_prefill(cfg_text_context, seg5_str) + + # GEN LATENT — at OUTPUT shape (not ref shape). Place at rope + # starting from rope_before_vae (the modality==1≡2 trick). The KV + # cache still holds the ref VAE at its original (smaller) rope + # range; the noise query's own rope just starts at the same + # ``rope_before_vae`` and extends through num_vid_tokens. + gen_input_lat = self.bagel.prepare_video_latent( + curr_kvlens=gen_context["kv_lens"], + curr_rope=[rope_before_vae], + video_shapes=[out_shape], + new_token_ids=self.new_token_ids, + ) + for k, v in gen_input_lat.items(): + if torch.is_tensor(v): + gen_input_lat[k] = v.to(self.device) + cfg_text_lat = self.bagel.prepare_video_latent_cfg( + curr_kvlens=cfg_text_context["kv_lens"], + curr_rope=[cfg_rope_before_vae], + video_shapes=[out_shape], + ) + for k, v in cfg_text_lat.items(): + if torch.is_tensor(v): + cfg_text_lat[k] = v.to(self.device) + cfg_img_lat = cfg_text_lat + + self._regen_init_noise_on_device(gen_input_lat, req.sampling_params.seed) + + # ── First-frame conditioning: VAE-encode the FULL pixel-space + # target tensor (matches upstream Lance PR #33's ff2v_sample): + # + # vae_tensor = torch.randn([3, num_frames, H, W]) + # vae_tensor[:, :4, :, :] = image.repeat(1, 4, 1, 1) + # + # Then VAE-encode the whole thing and take latent frame 0. This is + # important — Wan2.2 VAE has temporal convolutions, so the first + # latent frame's value depends on neighbouring pixel frames (not + # just frames 0-3). Encoding only the 4 image-replicated frames + # (which is what we did before) gives a *different* latent than + # upstream's full-sequence encode, and that delta produces an + # over-anchoring effect at i2v inference time (ex5 portrait case + # was visibly frozen because of this). + h_lat = int(out_H // self.bagel.latent_downsample) + w_lat = int(out_W // self.bagel.latent_downsample) + first_slice_n = h_lat * w_lat + + from torchvision.transforms.functional import resize as _tv_resize + + img_chw = torch.from_numpy(image_raw).permute(2, 0, 1).float() / 127.5 - 1.0 # (3, H, W) in [-1, 1] + img_chw = _tv_resize(img_chw, [out_H, out_W], antialias=True) + + # Build the full pixel-space target tensor: random noise everywhere, + # except the first 4 pixel frames are the input image (replicated). + # Wan2.2 VAE collapses every 4 pixel frames → 1 latent frame, so the + # first latent slice will represent the conditioning image. + full_pixel_tensor = torch.randn((3, num_frames_out, out_H, out_W), dtype=torch.float32, device=self.device) + full_pixel_tensor[:, :4, :, :] = img_chw.to(self.device).unsqueeze(1).repeat(1, 4, 1, 1) + full_pixel_5d = full_pixel_tensor.unsqueeze(0) # (1, 3, num_frames_out, H, W) + + with torch.autocast(**self._autocast_kwargs()): + image_latent_5d = self.vae.encode(full_pixel_5d) # (1, 48, T_lat, h_lat, w_lat) + + # Extract first latent frame in (h_lat, w_lat, 48) layout, then + # flatten to (h_lat*w_lat, 48) to match packed_init_noises[:first_slice_n]. + if image_latent_5d.dim() == 5: + image_latent = image_latent_5d[0, :, 0, :, :] # (48, h_lat, w_lat) + elif image_latent_5d.dim() == 4: + # encode may squeeze T=1 if the caller passed 4-D input — we + # passed 5-D so this branch shouldn't fire, but keep defensive. + image_latent = image_latent_5d[0] + else: + raise RuntimeError(f"Unexpected VAE-encoded shape {tuple(image_latent_5d.shape)}") + image_latent_packed = image_latent.permute(1, 2, 0).reshape(first_slice_n, -1) + + init_noises = gen_input_lat["packed_init_noises"] + image_latent_packed = image_latent_packed.to(device=init_noises.device, dtype=init_noises.dtype) + init_noises[:first_slice_n] = image_latent_packed + gen_input_lat["packed_init_noises"] = init_noises + + frame_condition_token_indexes = torch.arange(first_slice_n, dtype=torch.long, device=self.device) + + logger.info( + "Lance i2v cond: latent grid h_lat=%d w_lat=%d -> pinning first %d tokens (frame 0) at VAE-encoded image", + h_lat, + w_lat, + first_slice_n, + ) + + with torch.autocast(**self._autocast_kwargs()): + # ``prepare_video_latent`` / ``prepare_vae_latent`` still return + # ``key_values_lens`` / ``packed_indexes`` / ``packed_key_value_indexes`` + # but post-main-merge ``generate_image`` doesn't accept them — drop + # before unpacking via ``**gen_input_lat``. + for _drop in ("key_values_lens", "packed_indexes", "packed_key_value_indexes"): + gen_input_lat.pop(_drop, None) + latents, *_ = self.bagel.generate_image( + past_key_values=gen_context["past_key_values"], + cfg_text_past_key_values=cfg_text_context["past_key_values"] + if cfg_text_scale > 1.0 + else gen_context["past_key_values"], + cfg_img_past_key_values=gen_context["past_key_values"], + num_timesteps=num_timesteps, + timestep_shift=timestep_shift, + cfg_text_scale=cfg_text_scale, + cfg_img_scale=cfg_img_scale, + cfg_interval=cfg_interval, + cfg_renorm_type=cfg_renorm_type, + cfg_renorm_min=cfg_renorm_min, + **gen_input_lat, + cfg_text_packed_position_ids=cfg_text_lat["cfg_packed_position_ids"], + # ``cfg_text_packed_query_indexes`` / ``cfg_text_key_values_lens`` + # / ``cfg_text_packed_key_value_indexes`` removed post-main-merge + # (derived from ``cfg_text_past_key_values``). + cfg_img_packed_position_ids=cfg_img_lat["cfg_packed_position_ids"], + # ``cfg_img_*`` index/lens kwargs removed — same as above. + frame_condition_token_indexes=frame_condition_token_indexes, + ) + + frames_np = self._decode_video_from_latent(self.bagel, self.vae, latents[0], out_shape) + frames = [Image.fromarray(f) for f in frames_np] + return DiffusionOutput( + output=frames[0], + custom_output={"video_frames": frames, "video_shape": out_shape}, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ) + + def _forward_video_edit(self, req): + """Video edit: reference video + text prompt → modified video. + + Mirrors :meth:`_forward_image_edit`'s segmented prefill layout with + a temporal axis threaded through both prefill and the gen latent. + Verified against upstream Lance's ``Lance.validation_gen_KVcache`` for + ``--task video_edit --resolution video_480p``: + + seg1 (causal, modality=system_prompt): + ``<|im_start|>system\\n{sys}<|im_end|>\\n<|im_start|>user\\n`` + seg2 (full, modality=ref_vit): + ViT(video) — ref video through Qwen2.5-VL ViT, t-axis +1000 + seg3 (full_noise, modality=ref_source): + VAE(video) — ref video through Wan2.2 VAE, time_embed(t=0). + Rope shares range with the gen noise latent (modality 1==2 + trick) so the model can align ref↔output per ``(t,h,w)``. + seg4 (causal, modality=text): the user instruction + seg5 (causal, modality=system_prompt): + ``<|im_end|>\\n<|im_start|>assistant\\n`` + seg6 (noise QUERY): 3-D gen latent at the SAME rope as seg3. + + cfg_text_context skips ONLY seg4 (user text), matching upstream's + ``uncond_split_pro_kvcache``. cfg_img defaults to off + (``cfg_img_scale=1.0`` → upstream's ``cfg_vit_scale=1.0``). + + Replaces the previous shifted-rope layout (``_EDIT_SHIFT=10000``) + which kept ViT/text at a far-away rope range — that caused 15.5% + rel_l2 divergence at layer 0 output (cond) vs upstream because the + noise query attended to a cached K/V at rope (0..47) while upstream + attends at rope (111..159). + """ + + import numpy as _np + + from vllm_omni.diffusion.data import DiffusionOutput + + from .lance_transformer import NaiveCache + + first_prompt = req.prompts[0] + assert isinstance(first_prompt, dict), "video_edit requires dict-style prompt" + user_text = first_prompt.get("user_text") + rendered = first_prompt.get("prompt") or "" + if not user_text: + user_text = _extract_user_instruction(rendered) + mm_data = first_prompt.get("multi_modal_data") or {} + video_input = mm_data.get("video") + if video_input is None: + raise ValueError("video_edit requires multi_modal_data.video.") + # Resolve to raw (T, H, W, 3) uint8 ndarray + origin_fps so the + # upstream-style bucket resize + frame sampler can replicate the + # exact preprocessing path. If the caller passed an already-decoded + # ndarray with no FPS, default to 24fps (matches the official + # video_edit examples). + extra_args = first_prompt.get("extra_args") or {} + sp_extra = getattr(req.sampling_params, "extra_args", {}) or {} + extra_args = {**extra_args, **sp_extra} + origin_fps_default = float(extra_args.get("origin_fps", 24.0)) + # Path-only fast path: use decord (matches upstream Lance's + # ``data/datasets_custom/validation_dataset.py`` which decodes via + # ``decord.VideoReader``). decord vs cv2 produce different YUV→RGB + # conversions (~0.1 RGB units of max diff), and that propagates into + # both the VAE and ViT pixel inputs. Falling back to cv2 only when + # decord is unavailable keeps single-machine parity with upstream + # while not requiring decord in production. + if isinstance(video_input, str): + try: + import decord as _decord + from decord import VideoReader as _VideoReader # type: ignore + + vr = _VideoReader(video_input, ctx=_decord.cpu(0)) + origin_fps = float(vr.get_avg_fps()) or origin_fps_default + idx_all = list(range(len(vr))) + video_raw = vr.get_batch(idx_all).asnumpy() # (T, H, W, 3) uint8 RGB + except ImportError: + import cv2 as _cv2 + + cap = _cv2.VideoCapture(video_input) + origin_fps = float(cap.get(_cv2.CAP_PROP_FPS) or origin_fps_default) + frames_bgr = [] + while True: + ok, f = cap.read() + if not ok: + break + frames_bgr.append(_cv2.cvtColor(f, _cv2.COLOR_BGR2RGB)) + cap.release() + video_raw = _np.stack(frames_bgr, axis=0) + elif isinstance(video_input, _np.ndarray): + video_raw = video_input + origin_fps = origin_fps_default + elif torch.is_tensor(video_input): + arr = video_input.detach().cpu().numpy() + if arr.max() <= 1.5: + arr = (arr * 255.0).clip(0, 255).astype(_np.uint8) + video_raw = arr + origin_fps = origin_fps_default + else: + raise ValueError(f"Unsupported video_input type {type(video_input)}") + + cfg_text_scale = float(extra_args.get("cfg_text_scale", 4.0)) + cfg_img_scale = float(extra_args.get("cfg_img_scale", 1.0)) + # Upstream Lance default is ``cfg_interval=[0, 1]`` (CFG active on every + # denoise step). vllm-omni's previous (0.4, 1.0) skipped CFG for the + # last ~10 iterations after timestep_shift=3.5, which contributed + # most of the late-trajectory drift from upstream. + cfg_interval = tuple(extra_args.get("cfg_interval", (0.0, 1.0))) + cfg_renorm_type = str(extra_args.get("cfg_renorm_type", "global")) + cfg_renorm_min = float(extra_args.get("cfg_renorm_min", 0.0)) + timestep_shift = float(extra_args.get("timestep_shift", LANCE_DEFAULTS.timestep_shift)) + num_timesteps = int(req.sampling_params.num_inference_steps or LANCE_DEFAULTS.num_timesteps) + + # Run upstream's BucketResize + frame sampler so VAE / ViT see + # byte-identical pixels to the reference implementation. Without this, + # vllm-omni's ViT grid was (T=23, H=40, W=54) for the car video versus + # upstream's (T=23, H=36, W=50) — different K/V at ViT positions + # accounted for the residual ~9% rel_l2 at layer 0 output. + from .lance_transformer import LanceBagel as _LanceBagel + + vae_video, vit_pixels, vit_grid_thw, sampled_T = _LanceBagel._lance_video_preprocess(video_raw, origin_fps) + video_chw = vae_video # (C, T_sampled, H_vae, W_vae) — model-space [-1,1] + T_sampled = int(video_chw.shape[1]) + H_vae = int(video_chw.shape[2]) + W_vae = int(video_chw.shape[3]) + video_shape = (T_sampled, H_vae, W_vae) + logger.info( + "Lance video_edit: input %dx%dx%d → sampled %dx%dx%d (T,H,W) @ %.2ffps, ViT grid_thw=%s, user_text=%r", + video_raw.shape[0], + video_raw.shape[1], + video_raw.shape[2], + T_sampled, + H_vae, + W_vae, + origin_fps, + tuple(vit_grid_thw[0].tolist()), + user_text, + ) + + if req.sampling_params.seed is not None: + torch.manual_seed(req.sampling_params.seed) + if self.device.type == "cuda": + torch.cuda.manual_seed(req.sampling_params.seed) + + gen_context = { + "kv_lens": [0], + "ropes": [0], + "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), + } + cfg_text_context = { + "kv_lens": [0], + "ropes": [0], + "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), + } + + # Shared latent cache so gen + cfg branches see the same VAE sample. + ref_latent_cache: list = [] + + def _vae_transforms(_): + return video_chw + + # ----- prefill segments in upstream order ----- + seg1_str, seg5_str = self._segment_strings("video_edit", "video") + + # seg1: system + user header (shared) + self._raw_text_prefill(gen_context, seg1_str) + if cfg_text_scale > 1.0: + self._raw_text_prefill(cfg_text_context, seg1_str) + + # seg2: ViT(ref) (shared) — feed pre-bucketed patches directly so + # the Qwen2VLImageProcessor smart-resize is bypassed. + precomputed_vit = [(vit_pixels, vit_grid_thw)] + self._vit_video_prefill(gen_context, [video_raw], precomputed_vit=precomputed_vit) + if cfg_text_scale > 1.0: + self._vit_video_prefill(cfg_text_context, [video_raw], precomputed_vit=precomputed_vit) + + # seg3: VAE(ref) — snapshot rope BEFORE the prefill so the gen noise + # latent can be placed at the SAME rope range (matches upstream's + # modality==1≡2 trick). + rope_before_vae = gen_context["ropes"][0] + cfg_rope_before_vae = cfg_text_context["ropes"][0] if cfg_text_scale > 1.0 else rope_before_vae + self._vae_ref_prefill(gen_context, [video_chw], _vae_transforms, is_video=True, latent_cache=ref_latent_cache) + if cfg_text_scale > 1.0: + self._vae_ref_prefill( + cfg_text_context, [video_chw], _vae_transforms, is_video=True, latent_cache=ref_latent_cache + ) + + # seg4: user instruction (gen only — cfg branch skips per upstream). + self._raw_text_prefill(gen_context, user_text) + + # seg5: separator + assistant header (both gen and cfg, each at its + # own current rope; cfg's lands earlier than gen's because cfg + # skipped seg4). + self._raw_text_prefill(gen_context, seg5_str) + if cfg_text_scale > 1.0: + self._raw_text_prefill(cfg_text_context, seg5_str) + + # seg6 (noise QUERY): gen latent at the SAME rope as the ref VAE + # block. KV cache still contains sys+ViT+VAE_ref+user_text+sep at + # their post-prefill rope positions; only the noise query's Q rope + # is rewound to overlap with the VAE_ref block. + gen_input_lat = self.bagel.prepare_video_latent( + curr_kvlens=gen_context["kv_lens"], + curr_rope=[rope_before_vae], + video_shapes=[video_shape], + new_token_ids=self.new_token_ids, + ) + for k, v in gen_input_lat.items(): + if torch.is_tensor(v): + gen_input_lat[k] = v.to(self.device) + cfg_text_lat = self.bagel.prepare_video_latent_cfg( + curr_kvlens=cfg_text_context["kv_lens"], + curr_rope=[cfg_rope_before_vae], + video_shapes=[video_shape], + ) + for k, v in cfg_text_lat.items(): + if torch.is_tensor(v): + cfg_text_lat[k] = v.to(self.device) + + # cfg_img branch is off (cfg_img_scale=1.0); generate_image needs the + # arg surface but skips the branch when scale<=1.0. + cfg_img_lat = cfg_text_lat + + self._regen_init_noise_on_device(gen_input_lat, req.sampling_params.seed) + + with torch.autocast(**self._autocast_kwargs()): + # ``prepare_video_latent`` / ``prepare_vae_latent`` still return + # ``key_values_lens`` / ``packed_indexes`` / ``packed_key_value_indexes`` + # but post-main-merge ``generate_image`` doesn't accept them — drop + # before unpacking via ``**gen_input_lat``. + for _drop in ("key_values_lens", "packed_indexes", "packed_key_value_indexes"): + gen_input_lat.pop(_drop, None) + latents, *_ = self.bagel.generate_image( + past_key_values=gen_context["past_key_values"], + cfg_text_past_key_values=cfg_text_context["past_key_values"] + if cfg_text_scale > 1.0 + else gen_context["past_key_values"], + cfg_img_past_key_values=gen_context["past_key_values"], + num_timesteps=num_timesteps, + timestep_shift=timestep_shift, + cfg_text_scale=cfg_text_scale, + cfg_img_scale=cfg_img_scale, + cfg_interval=cfg_interval, + cfg_renorm_type=cfg_renorm_type, + cfg_renorm_min=cfg_renorm_min, + **gen_input_lat, + cfg_text_packed_position_ids=cfg_text_lat["cfg_packed_position_ids"], + # ``cfg_text_packed_query_indexes`` / ``cfg_text_key_values_lens`` + # / ``cfg_text_packed_key_value_indexes`` removed post-main-merge + # (derived from ``cfg_text_past_key_values``). + cfg_img_packed_position_ids=cfg_img_lat["cfg_packed_position_ids"], + # ``cfg_img_*`` index/lens kwargs removed — same as above. + ) + + frames_np = self._decode_video_from_latent(self.bagel, self.vae, latents[0], video_shape) + frames = [Image.fromarray(f) for f in frames_np] + return DiffusionOutput( + output=frames[0], + custom_output={"video_frames": frames, "video_shape": video_shape}, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ) + + @torch.inference_mode() + def _forward_x2t_video(self, req): + """Video understanding (x2t_video): video → text caption / VQA answer. + + Builds the prefill in three segments matching upstream's chat + template — see :meth:`_forward_x2t_image` for the rationale on + why naïve concatenation (text prompt → ViT) lands the EOS / vit + markers in the wrong place and degrades the answer format. + """ + + from vllm_omni.diffusion.data import DiffusionOutput + + from .lance_transformer import NaiveCache + + first_prompt = req.prompts[0] + assert isinstance(first_prompt, dict), "x2t_video requires dict-style prompt" + video_input = (first_prompt.get("multi_modal_data") or {}).get("video") + if video_input is None: + raise ValueError("x2t_video requires multi_modal_data.video to be a video tensor/array/path.") + # Accept a path → load via imageio; else assume tensor / numpy array (T, H, W, 3). + if isinstance(video_input, str): + import imageio.v3 as iio + + video_input = iio.imread(video_input) + # Wrap single video as a list for the prep function. + videos = [video_input] + + extra_args = first_prompt.get("extra_args") or {} + sp_extra = getattr(req.sampling_params, "extra_args", {}) or {} + extra_args = {**extra_args, **sp_extra} + max_text_tokens = int(extra_args.get("max_think_tokens", 200)) + do_sample = bool(extra_args.get("do_sample", False)) + text_temperature = float(extra_args.get("text_temperature", 0.3)) + + prefix_text, suffix_text = self._split_x2t_prompt( + first_prompt.get("prompt") or "", + user_text_override=extra_args.get("user_text"), + system_prompt_override=extra_args.get("system_prompt"), + default_system_prompt="Watch the video carefully and answer the question.", + ) + + if req.sampling_params.seed is not None: + torch.manual_seed(req.sampling_params.seed) + if self.device.type == "cuda": + torch.cuda.manual_seed(req.sampling_params.seed) + + gen_context = { + "kv_lens": [0], + "ropes": [0], + "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), + } + + self._raw_text_prefill(gen_context, prefix_text) + self._vit_video_prefill(gen_context, videos) + self._raw_text_prefill(gen_context, suffix_text) + + start_input = self._x2t_prepare_assistant_start(gen_context) + for k, v in start_input.items(): + if torch.is_tensor(v): + start_input[k] = v.to(self.device) + with torch.autocast(**self._autocast_kwargs()): + token_ids = self.bagel.generate_text( + past_key_values=gen_context["past_key_values"], + max_length=max_text_tokens, + do_sample=do_sample, + temperature=text_temperature, + end_token_id=self.new_token_ids["eos_token_id"], + **start_input, + ) + decoded = self.tokenizer.decode(token_ids[:, 0].tolist()) + text_output = decoded.split("<|im_end|>")[0] + if "<|im_start|>" in text_output: + text_output = text_output.split("<|im_start|>")[-1] + # Strip the leading newline that was fed as the start token (see + # ``_x2t_prepare_assistant_start``); see x2t_image for rationale. + text_output = text_output.lstrip("\n") + logger.info("Lance x2t_video: generated %d tokens", token_ids.shape[0]) + return DiffusionOutput( + output=text_output, + custom_output={"text_output": text_output}, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ) + + @torch.inference_mode() + def _forward_x2t_image(self, req): + """Image understanding (x2t_image): image + question → text answer. + + Builds the prefill in three text/vit segments matching upstream's + chat-template layout:: + + <|im_start|>system\\n{system}<|im_end|>\\n<|im_start|>user\\n + <|vision_start|>{N×ViT}<|vision_end|> + {question}<|im_end|>\\n<|im_start|>assistant\\n + + ``prepare_prompts`` wraps the *whole* string with ``[bos]…[eos]`` + and ``prepare_vit_images`` adds ``<|vision_start|>``/``<|vision_end|>`` + of its own — concatenating them naïvely (text first, then vit) + produced ``…assistant\\n[eos]<|vision_start|>{ViT}<|vision_end|>``, + so the model saw an EOS right after the assistant tag and a + bare ``<|video_pad|>`` token inside the user message. Segmenting + the text into prefix/suffix and sliding the vit prefill between + them puts the markers exactly where Lance was trained on. + """ + + from PIL import Image as _PIL_Image + + from vllm_omni.diffusion.data import DiffusionOutput + + from .lance_transformer import NaiveCache + + first_prompt = req.prompts[0] + assert isinstance(first_prompt, dict), "x2t_image requires dict-style prompt" + image_input = (first_prompt.get("multi_modal_data") or {}).get("image") + if image_input is None: + raise ValueError("x2t_image requires multi_modal_data.image to be a PIL image / array / path.") + if isinstance(image_input, str): + image_input = _PIL_Image.open(image_input).convert("RGB") + images = [image_input] + + extra_args = first_prompt.get("extra_args") or {} + sp_extra = getattr(req.sampling_params, "extra_args", {}) or {} + extra_args = {**extra_args, **sp_extra} + max_text_tokens = int(extra_args.get("max_think_tokens", 256)) + do_sample = bool(extra_args.get("do_sample", False)) + text_temperature = float(extra_args.get("text_temperature", 0.3)) + + prefix_text, suffix_text = self._split_x2t_prompt( + first_prompt.get("prompt") or "", + user_text_override=extra_args.get("user_text"), + system_prompt_override=extra_args.get("system_prompt"), + default_system_prompt="Look at the image carefully and answer the question.", + ) + + if req.sampling_params.seed is not None: + torch.manual_seed(req.sampling_params.seed) + if self.device.type == "cuda": + torch.cuda.manual_seed(req.sampling_params.seed) + + def vit_transforms(img): + return self.image_processor(images=img, return_tensors="pt").pixel_values[0] + + gen_context = { + "kv_lens": [0], + "ropes": [0], + "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), + } + + self._raw_text_prefill(gen_context, prefix_text) + self._vit_image_prefill(gen_context, images, vit_transforms) + self._raw_text_prefill(gen_context, suffix_text) + + start_input = self._x2t_prepare_assistant_start(gen_context) + for k, v in start_input.items(): + if torch.is_tensor(v): + start_input[k] = v.to(self.device) + with torch.autocast(**self._autocast_kwargs()): + token_ids = self.bagel.generate_text( + past_key_values=gen_context["past_key_values"], + max_length=max_text_tokens, + do_sample=do_sample, + temperature=text_temperature, + end_token_id=self.new_token_ids["eos_token_id"], + **start_input, + ) + decoded = self.tokenizer.decode(token_ids[:, 0].tolist()) + text_output = decoded.split("<|im_end|>")[0] + if "<|im_start|>" in text_output: + text_output = text_output.split("<|im_start|>")[-1] + # Strip the leading newline that was fed as the start token (see + # ``_x2t_prepare_assistant_start``); it always sits at index 0 of + # the generated sequence and is just plumbing, not model output. + text_output = text_output.lstrip("\n") + logger.info("Lance x2t_image: generated %d tokens", token_ids.shape[0]) + return DiffusionOutput( + output=text_output, + custom_output={"text_output": text_output}, + stage_durations=self.stage_durations if hasattr(self, "stage_durations") else None, + ) + + @staticmethod + def _split_x2t_prompt( + rendered_prompt: str, + *, + user_text_override: str | None, + system_prompt_override: str | None, + default_system_prompt: str, + ) -> tuple[str, str]: + """Split a rendered Lance x2t chat template at the vision block. + + Accepts either a pre-rendered template containing + ``<|vision_start|>…<|vision_end|>`` (the historical + ``render_lance_prompt`` output) or recovers prefix/suffix from + ``user_text_override`` + ``system_prompt_override`` when the + template was not pre-rendered. Returns ``(prefix, suffix)`` such + that the segmented prefill is ``prefix + ViT + suffix``. + + ``suffix`` deliberately ends at ``assistant`` (NOT ``assistant\\n``): + :meth:`Bagel.prepare_start_tokens` injects ``<|im_start|>`` (Qwen + bos == 151644) as the first generation step, and ``\\n<|im_start|>`` + right after ``assistant\\n`` tells the model "begin a new + message" — it answers with an immediate ``<|im_end|>`` and + produces an empty caption. Holding the trailing ``\\n`` out of + the prefill (it gets re-applied via ``_x2t_prepare_assistant_start``) + lets the next forward step start *inside* the assistant turn + instead of opening a fresh one. + """ + sys_prompt = system_prompt_override or default_system_prompt + + if user_text_override is not None: + user_text = user_text_override + elif "<|vision_start|>" in rendered_prompt and "<|vision_end|>" in rendered_prompt: + _, _, after_vis_end = rendered_prompt.partition("<|vision_end|>") + user_text, _, _ = after_vis_end.partition("<|im_end|>") + else: + # No vision markers and no override — best-effort fallback: + # treat the entire prompt as the user question and rebuild + # the chat scaffolding. + user_text = rendered_prompt + + prefix = f"<|im_start|>system\n{sys_prompt}<|im_end|>\n<|im_start|>user\n" + suffix = f"{user_text}<|im_end|>\n<|im_start|>assistant" + return prefix, suffix + + def _x2t_prepare_assistant_start(self, gen_context): + """Build a ``generate_text`` start input where the first fed token + is the newline that closes ``<|im_start|>assistant\\n``. + + :meth:`Bagel.prepare_start_tokens` would instead feed + ``<|im_start|>``, opening a fresh message right after the + assistant tag and triggering an immediate EOS on Lance_3B (the + image-only checkpoint is stricter about chat-turn framing than + the video model, which happens to recover). Feeding ``\\n`` + keeps the model *inside* the assistant turn so the next argmax + is the first answer token. + """ + newline_ids = self.tokenizer.encode("\n", add_special_tokens=False) + assert len(newline_ids) == 1, f"expected single token for '\\n', got {newline_ids!r}" + nl_id = newline_ids[0] + + curr_kvlens = gen_context["kv_lens"] + curr_ropes = gen_context["ropes"] + packed_key_value_indexes: list[int] = [] + packed_query_position_ids: list[int] = [] + packed_start_tokens: list[int] = [] + curr = 0 + for curr_kvlen, curr_position_id in zip(curr_kvlens, curr_ropes): + packed_key_value_indexes.extend(range(curr, curr + curr_kvlen)) + packed_start_tokens.append(nl_id) + packed_query_position_ids.append(curr_position_id) + curr += curr_kvlen + # ``Bagel.generate_text`` post-main-merge no longer accepts + # ``key_values_lens`` / ``packed_key_value_indexes`` — they're + # derived from ``past_key_values``. + return { + "packed_start_tokens": torch.tensor(packed_start_tokens, dtype=torch.long), + "packed_query_position_ids": torch.tensor(packed_query_position_ids, dtype=torch.long), + } + + @staticmethod + def _decode_video_from_latent( + bagel, + vae: LanceWanVAE, + latent: torch.Tensor, + video_shape: tuple[int, int, int], + ): + """Pack a flat Lance latent into ``(B, 48, t, h, w)`` and decode to a + ``(T_lat, H, W, 3)`` numpy frame list via :meth:`LanceWanVAE.decode_video`. + """ + + T, H, W = video_shape + downsample_t = int(getattr(bagel.config.vae_config, "downsample_temporal", 4)) + t_lat = (T - 1) // downsample_t + 1 + h_lat = H // bagel.latent_downsample + w_lat = W // bagel.latent_downsample + p = bagel.latent_patch_size + c = bagel.latent_channel + # ``latent`` is flat ``(t_lat*h_lat*w_lat, c * p * p)`` (Lance image + # path uses ``p=1`` so this is ``(N, c)``). Reshape into the 5-D layout + # the Wan2.2 VAE decoder consumes. + latent = latent.reshape(t_lat, h_lat, w_lat, p, p, c) + latent = torch.einsum("thwpqc->cthpwq", latent) + latent = latent.reshape(1, c, t_lat, h_lat * p, w_lat * p) + + vae_dtype = next(vae.parameters()).dtype + latent = latent.to(vae_dtype) + video = vae.decode_video(latent) # (1, 3, T_out, H_out, W_out) + # Scale [-1,1] -> [0,255] uint8 frames; the upstream VAE already clamps. + video = (video[0] * 0.5 + 0.5).clamp(0, 1).permute(1, 2, 3, 0) * 255 + return video.to(torch.uint8).cpu().numpy() # (T_out, H_out, W_out, 3) + + def _build_wan22_vae(self, repo_root: str) -> LanceWanVAE: + """Wrap the bundled ``Wan2.2_VAE.pth`` with a BAGEL-compatible + ``.encode(images)`` / ``.decode(latent)`` surface. + + Uses the upstream Wan2.2 VAE module (ported in :mod:`wan_vae`) since + the released ``.pth`` is keyed for that module — *not* for the diffusers + ``AutoencoderKLWan`` state dict that ``OmniAutoencoderKLWan`` expects. + Single images are treated as 1-frame clips; video latents pass through + the same module via the 5-D ``encode_video``/``decode_video`` path. + Wan2.2 VAE constants: z=48ch, /16 spatial, /4 temporal. + """ + vae_path = os.path.join(repo_root, _VAE_FILE) + return LanceWanVAE(vae_path=vae_path, dtype=torch.bfloat16, device=self.device) diff --git a/vllm_omni/diffusion/models/lance/prompts.py b/vllm_omni/diffusion/models/lance/prompts.py new file mode 100644 index 000000000..efdd007e8 --- /dev/null +++ b/vllm_omni/diffusion/models/lance/prompts.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Lance chat / system prompts. + +Matches the upstream Lance training distribution (see +``bytedance/Lance/data/system_prompt_render.py`` and +``data/common.py::generate_system_prompt``): + + * each task has a specific system prompt describing what to attend to; + * the prompt is wrapped in the Qwen chat template + ``<|im_start|>system\\n…<|im_end|>\\n<|im_start|>user\\n…<|im_end|>\\n`` + ``<|im_start|>assistant\\n``; + * vision tokens are framed by ``<|vision_start|><|video_pad|><|vision_end|>`` + (upstream uses ``<|video_pad|>`` even for image type by default). + +These are pure string formatting helpers — the runtime pipeline still does +the actual VAE / ViT prefill separately. +""" + +from __future__ import annotations + +# Task-specific system prompts (verbatim from upstream's +# ``generate_system_prompt`` — first variant of each pool). +SYSTEM_PROMPTS: dict[tuple[str, str], str] = { + # (task, vision_type) -> system prompt + ("t2i", "image"): ( + "Describe the image by detailing the color, quantity, text, shape, " + "size, texture, spatial relationships of the objects and background:" + ), + ("t2v", "video"): ( + "Describe the video by detailing the color, quantity, visible text, " + "shape, size, texture, spatial relationships and motion/camera " + "movements of the objects and background:" + ), + # i2v reuses the t2v system prompt (matches upstream Lance PR #33's + # ``data/common.py`` where ``system_prompt_type in ("t2v", "i2v")`` + # selects the same prompt pool). + ("i2v", "video"): ( + "Describe the video by detailing the color, quantity, visible text, " + "shape, size, texture, spatial relationships and motion/camera " + "movements of the objects and background:" + ), + ("image_edit", "image"): ( + "Describe the key features of the input image (color, shape, size, " + "texture, objects, background), then explain how the user's text " + "instruction should alter or modify the image. Generate a new image " + "that meets the user's requirements while maintaining consistency " + "with the original input where appropriate." + ), + ("video_edit", "video"): ( + "Describe the key features of the input video (color, shape, size, " + "texture, objects, background), then explain how the user's text " + "instruction should alter or modify the video. Generate a new video " + "that meets the user's requirements while maintaining consistency " + "with the original input where appropriate." + ), + ("x2t_image", "image"): ( + "Generate a detailed and accurate description of the image, including all the key moments and visual details." + ), + ("x2t_video", "video"): ( + "Generate a detailed and accurate description of the video, including all the key moments and visual details." + ), +} + +# Vision-content placeholders (Qwen2.5-VL convention). Upstream uses +# ``<|video_pad|>`` for image inputs too unless ``force_video_pad=False`` +# (counter-intuitively named; see ``render_qwenvl_prompt`` upstream). +VISION_START = "<|vision_start|>" +VISION_END = "<|vision_end|>" +VIDEO_PAD = "<|video_pad|>" +IMAGE_PAD = "<|image_pad|>" + + +def render_lance_prompt( + task: str, + user_text: str, + *, + vision_token: str | None = None, + system_prompt: str | None = None, +) -> str: + """Render a Lance-compatible user-side prompt. + + ``task`` is one of ``t2i``, ``t2v``, ``image_edit``, ``video_edit``, + ``x2t_image``, ``x2t_video``. ``vision_token`` should be the visual + placeholder to embed inside the user message (e.g. ``VIDEO_PAD`` or a + ``VISION_START + VIDEO_PAD + VISION_END`` block); pass ``None`` for + text-only inputs. ``system_prompt`` overrides the default task system + prompt — required for x2t QA examples whose upstream JSON carries a + per-example instruction (e.g. ``"Look at the image carefully and answer + the question."``); without it x2t falls back to the caption-style + default and the model describes instead of answering. + + Returns a single string ready to be tokenized; no further wrapping is + needed by the caller. + """ + # ``t2v`` / ``x2t_video`` / ``video_edit`` use video system prompts; + # the rest are image. + vision_type = "video" if task in {"t2v", "i2v", "x2t_video", "video_edit"} else "image" + sys_prompt = ( + system_prompt if system_prompt else SYSTEM_PROMPTS.get((task, vision_type), "You are a helpful assistant.") + ) + + if vision_token is None: + user_msg = user_text + else: + # Upstream concatenates the vision block immediately before the text. + user_msg = f"{vision_token}{user_text}" + + return ( + f"<|im_start|>system\n{sys_prompt}<|im_end|>\n<|im_start|>user\n{user_msg}<|im_end|>\n<|im_start|>assistant\n" + ) + + +__all__ = [ + "IMAGE_PAD", + "SYSTEM_PROMPTS", + "VIDEO_PAD", + "VISION_END", + "VISION_START", + "render_lance_prompt", +] diff --git a/vllm_omni/diffusion/models/lance/wan_vae.py b/vllm_omni/diffusion/models/lance/wan_vae.py new file mode 100644 index 000000000..a8e167534 --- /dev/null +++ b/vllm_omni/diffusion/models/lance/wan_vae.py @@ -0,0 +1,870 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Ported from ByteDance Lance upstream (https://github.com/bytedance/Lance, +# modeling/vae/wan/{vae2_2.py,model.py}). Upstream copyright: +# +# Copyright (c) 2025 ByteDance Ltd. and/or its affiliates. +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. +# Licensed under the Apache License, Version 2.0. +# +# Lance ships its VAE as a raw ``Wan2.2_VAE.pth`` whose state dict is keyed for +# the upstream ``WanVAE_`` nn.Module here (not for diffusers' ``AutoencoderKLWan``). +# We port that module verbatim so we can load the .pth directly, and wrap it with +# :class:`LanceWanVAE` which exposes BAGEL's 4-D ``encode(BCHW)/decode(BCHW)`` +# surface (and a 5-D path for the video checkpoint). +"""Wan2.2 VAE used by Lance, ported from upstream so ``Wan2.2_VAE.pth`` loads +natively without state-dict surgery.""" + +from __future__ import annotations + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange +from vllm.logger import init_logger + +logger = init_logger(__name__) + +CACHE_T = 2 + + +class CausalConv3d(nn.Conv3d): + """Causal 3D conv with feature-map caching across temporal chunks.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._padding = ( + self.padding[2], + self.padding[2], + self.padding[1], + self.padding[1], + 2 * self.padding[0], + 0, + ) + self.padding = (0, 0, 0) + + def forward(self, x, cache_x=None): + padding = list(self._padding) + if cache_x is not None and self._padding[4] > 0: + cache_x = cache_x.to(x.device) + x = torch.cat([cache_x, x], dim=2) + padding[4] -= cache_x.shape[2] + x = F.pad(x, padding) + return super().forward(x) + + +class RMS_norm(nn.Module): + def __init__(self, dim, channel_first=True, images=True, bias=False): + super().__init__() + broadcastable_dims = (1, 1, 1) if not images else (1, 1) + shape = (dim, *broadcastable_dims) if channel_first else (dim,) + self.channel_first = channel_first + self.scale = dim**0.5 + self.gamma = nn.Parameter(torch.ones(shape)) + self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0.0 + + def forward(self, x): + return F.normalize(x, dim=(1 if self.channel_first else -1)) * self.scale * self.gamma + self.bias + + +class Upsample(nn.Upsample): + def forward(self, x): + return super().forward(x.float()).type_as(x) + + +class Resample(nn.Module): + def __init__(self, dim, mode): + assert mode in ("none", "upsample2d", "upsample3d", "downsample2d", "downsample3d") + super().__init__() + self.dim = dim + self.mode = mode + + if mode == "upsample2d": + self.resample = nn.Sequential( + Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, dim, 3, padding=1), + ) + elif mode == "upsample3d": + self.resample = nn.Sequential( + Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, dim, 3, padding=1), + ) + self.time_conv = CausalConv3d(dim, dim * 2, (3, 1, 1), padding=(1, 0, 0)) + elif mode == "downsample2d": + self.resample = nn.Sequential(nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2))) + elif mode == "downsample3d": + self.resample = nn.Sequential(nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2))) + self.time_conv = CausalConv3d(dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0)) + else: + self.resample = nn.Identity() + + def forward(self, x, feat_cache=None, feat_idx=[0]): + b, c, t, h, w = x.size() + if self.mode == "upsample3d": + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = "Rep" + feat_idx[0] += 1 + else: + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None and feat_cache[idx] != "Rep": + cache_x = torch.cat( + [feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], + dim=2, + ) + if cache_x.shape[2] < 2 and feat_cache[idx] is not None and feat_cache[idx] == "Rep": + cache_x = torch.cat([torch.zeros_like(cache_x).to(cache_x.device), cache_x], dim=2) + if feat_cache[idx] == "Rep": + x = self.time_conv(x) + else: + x = self.time_conv(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + x = x.reshape(b, 2, c, t, h, w) + x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]), 3) + x = x.reshape(b, c, t * 2, h, w) + t = x.shape[2] + x = rearrange(x, "b c t h w -> (b t) c h w") + x = self.resample(x) + x = rearrange(x, "(b t) c h w -> b c t h w", t=t) + + if self.mode == "downsample3d": + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = x.clone() + feat_idx[0] += 1 + else: + cache_x = x[:, :, -1:, :, :].clone() + x = self.time_conv(torch.cat([feat_cache[idx][:, :, -1:, :, :], x], 2)) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + return x + + +class ResidualBlock(nn.Module): + def __init__(self, in_dim, out_dim, dropout=0.0): + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.residual = nn.Sequential( + RMS_norm(in_dim, images=False), + nn.SiLU(), + CausalConv3d(in_dim, out_dim, 3, padding=1), + RMS_norm(out_dim, images=False), + nn.SiLU(), + nn.Dropout(dropout), + CausalConv3d(out_dim, out_dim, 3, padding=1), + ) + self.shortcut = CausalConv3d(in_dim, out_dim, 1) if in_dim != out_dim else nn.Identity() + + def forward(self, x, feat_cache=None, feat_idx=[0]): + h = self.shortcut(x) + for layer in self.residual: + if isinstance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], + dim=2, + ) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x + h + + +class AttentionBlock(nn.Module): + """Single-head causal self-attention over spatial tokens, per frame.""" + + def __init__(self, dim): + super().__init__() + self.dim = dim + self.norm = RMS_norm(dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + self.proj = nn.Conv2d(dim, dim, 1) + nn.init.zeros_(self.proj.weight) + + def forward(self, x): + identity = x + b, c, t, h, w = x.size() + x = rearrange(x, "b c t h w -> (b t) c h w") + x = self.norm(x) + q, k, v = self.to_qkv(x).reshape(b * t, 1, c * 3, -1).permute(0, 1, 3, 2).contiguous().chunk(3, dim=-1) + x = F.scaled_dot_product_attention(q, k, v) + x = x.squeeze(1).permute(0, 2, 1).reshape(b * t, c, h, w) + x = self.proj(x) + x = rearrange(x, "(b t) c h w-> b c t h w", t=t) + return x + identity + + +def _patchify(x, patch_size): + if patch_size == 1: + return x + if x.dim() == 4: + return rearrange(x, "b c (h q) (w r) -> b (c r q) h w", q=patch_size, r=patch_size) + if x.dim() == 5: + return rearrange(x, "b c f (h q) (w r) -> b (c r q) f h w", q=patch_size, r=patch_size) + raise ValueError(f"Invalid input shape: {x.shape}") + + +def _unpatchify(x, patch_size): + if patch_size == 1: + return x + if x.dim() == 4: + return rearrange(x, "b (c r q) h w -> b c (h q) (w r)", q=patch_size, r=patch_size) + if x.dim() == 5: + return rearrange(x, "b (c r q) f h w -> b c f (h q) (w r)", q=patch_size, r=patch_size) + return x + + +class AvgDown3D(nn.Module): + def __init__(self, in_channels, out_channels, factor_t, factor_s=1): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.factor_t = factor_t + self.factor_s = factor_s + self.factor = self.factor_t * self.factor_s * self.factor_s + assert in_channels * self.factor % out_channels == 0 + self.group_size = in_channels * self.factor // out_channels + + def forward(self, x: torch.Tensor) -> torch.Tensor: + pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t + x = F.pad(x, (0, 0, 0, 0, pad_t, 0)) + B, C, T, H, W = x.shape + x = x.view( + B, + C, + T // self.factor_t, + self.factor_t, + H // self.factor_s, + self.factor_s, + W // self.factor_s, + self.factor_s, + ) + x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous() + x = x.view(B, C * self.factor, T // self.factor_t, H // self.factor_s, W // self.factor_s) + x = x.view(B, self.out_channels, self.group_size, T // self.factor_t, H // self.factor_s, W // self.factor_s) + return x.mean(dim=2) + + +class DupUp3D(nn.Module): + def __init__(self, in_channels: int, out_channels: int, factor_t, factor_s=1): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.factor_t = factor_t + self.factor_s = factor_s + self.factor = self.factor_t * self.factor_s * self.factor_s + assert out_channels * self.factor % in_channels == 0 + self.repeats = out_channels * self.factor // in_channels + + def forward(self, x: torch.Tensor, first_chunk=False) -> torch.Tensor: + x = x.repeat_interleave(self.repeats, dim=1) + x = x.view( + x.size(0), + self.out_channels, + self.factor_t, + self.factor_s, + self.factor_s, + x.size(2), + x.size(3), + x.size(4), + ) + x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous() + x = x.view( + x.size(0), + self.out_channels, + x.size(2) * self.factor_t, + x.size(4) * self.factor_s, + x.size(6) * self.factor_s, + ) + if first_chunk: + x = x[:, :, self.factor_t - 1 :, :, :] + return x + + +class Down_ResidualBlock(nn.Module): + def __init__(self, in_dim, out_dim, dropout, mult, temperal_downsample=False, down_flag=False): + super().__init__() + self.avg_shortcut = AvgDown3D( + in_dim, out_dim, factor_t=2 if temperal_downsample else 1, factor_s=2 if down_flag else 1 + ) + downsamples = [] + for _ in range(mult): + downsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + in_dim = out_dim + if down_flag: + mode = "downsample3d" if temperal_downsample else "downsample2d" + downsamples.append(Resample(out_dim, mode=mode)) + self.downsamples = nn.Sequential(*downsamples) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + x_copy = x.clone() + for module in self.downsamples: + x = module(x, feat_cache, feat_idx) + return x + self.avg_shortcut(x_copy) + + +class Up_ResidualBlock(nn.Module): + def __init__(self, in_dim, out_dim, dropout, mult, temporal_upsample=False, up_flag=False): + super().__init__() + if up_flag: + self.avg_shortcut = DupUp3D( + in_dim, out_dim, factor_t=2 if temporal_upsample else 1, factor_s=2 if up_flag else 1 + ) + else: + self.avg_shortcut = None + upsamples = [] + for _ in range(mult): + upsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + in_dim = out_dim + if up_flag: + mode = "upsample3d" if temporal_upsample else "upsample2d" + upsamples.append(Resample(out_dim, mode=mode)) + self.upsamples = nn.Sequential(*upsamples) + + def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False): + x_main = x.clone() + for module in self.upsamples: + x_main = module(x_main, feat_cache, feat_idx) + if self.avg_shortcut is not None: + return x_main + self.avg_shortcut(x, first_chunk) + return x_main + + +class Encoder3d(nn.Module): + def __init__( + self, + dim=128, + z_dim=4, + dim_mult=(1, 2, 4, 4), + num_res_blocks=2, + attn_scales=(), + temperal_downsample=(True, True, False), + dropout=0.0, + ): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = list(dim_mult) + self.num_res_blocks = num_res_blocks + self.attn_scales = list(attn_scales) + self.temperal_downsample = list(temperal_downsample) + + dims = [dim * u for u in [1] + self.dim_mult] + self.conv1 = CausalConv3d(12, dims[0], 3, padding=1) + + downsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + t_down_flag = self.temperal_downsample[i] if i < len(self.temperal_downsample) else False + downsamples.append( + Down_ResidualBlock( + in_dim=in_dim, + out_dim=out_dim, + dropout=dropout, + mult=num_res_blocks, + temperal_downsample=t_down_flag, + down_flag=i != len(self.dim_mult) - 1, + ) + ) + self.downsamples = nn.Sequential(*downsamples) + + self.middle = nn.Sequential( + ResidualBlock(out_dim, out_dim, dropout), + AttentionBlock(out_dim), + ResidualBlock(out_dim, out_dim, dropout), + ) + self.head = nn.Sequential( + RMS_norm(out_dim, images=False), + nn.SiLU(), + CausalConv3d(out_dim, z_dim, 3, padding=1), + ) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], + dim=2, + ) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + for layer in self.downsamples: + if feat_cache is not None: + x = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + for layer in self.middle: + if isinstance(layer, ResidualBlock) and feat_cache is not None: + x = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + for layer in self.head: + if isinstance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], + dim=2, + ) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x + + +class Decoder3d(nn.Module): + def __init__( + self, + dim=128, + z_dim=4, + dim_mult=(1, 2, 4, 4), + num_res_blocks=2, + attn_scales=(), + temporal_upsample=(False, True, True), + dropout=0.0, + ): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = list(dim_mult) + self.num_res_blocks = num_res_blocks + self.attn_scales = list(attn_scales) + self.temporal_upsample = list(temporal_upsample) + + dims = [dim * u for u in [self.dim_mult[-1]] + self.dim_mult[::-1]] + self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1) + self.middle = nn.Sequential( + ResidualBlock(dims[0], dims[0], dropout), + AttentionBlock(dims[0]), + ResidualBlock(dims[0], dims[0], dropout), + ) + + upsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + t_up_flag = self.temporal_upsample[i] if i < len(self.temporal_upsample) else False + upsamples.append( + Up_ResidualBlock( + in_dim=in_dim, + out_dim=out_dim, + dropout=dropout, + mult=num_res_blocks + 1, + temporal_upsample=t_up_flag, + up_flag=i != len(self.dim_mult) - 1, + ) + ) + self.upsamples = nn.Sequential(*upsamples) + self.head = nn.Sequential( + RMS_norm(out_dim, images=False), + nn.SiLU(), + CausalConv3d(out_dim, 12, 3, padding=1), + ) + + def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False): + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], + dim=2, + ) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + for layer in self.middle: + if isinstance(layer, ResidualBlock) and feat_cache is not None: + x = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + for layer in self.upsamples: + if feat_cache is not None: + x = layer(x, feat_cache, feat_idx, first_chunk) + else: + x = layer(x) + + for layer in self.head: + if isinstance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], + dim=2, + ) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x + + +def _count_conv3d(model): + return sum(1 for m in model.modules() if isinstance(m, CausalConv3d)) + + +class WanVAE_(nn.Module): + """Upstream Wan2.2 VAE module — encoder3d/decoder3d sandwich with 2x patchify + on input. State-dict-compatible with ``Wan2.2_VAE.pth``.""" + + def __init__( + self, + dim=160, + dec_dim=256, + z_dim=16, + dim_mult=(1, 2, 4, 4), + num_res_blocks=2, + attn_scales=(), + temperal_downsample=(True, True, False), + dropout=0.0, + ): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = list(dim_mult) + self.num_res_blocks = num_res_blocks + self.attn_scales = list(attn_scales) + self.temperal_downsample = list(temperal_downsample) + self.temporal_upsample = list(temperal_downsample)[::-1] + + self.encoder = Encoder3d( + dim, z_dim * 2, self.dim_mult, num_res_blocks, self.attn_scales, self.temperal_downsample, dropout + ) + self.conv1 = CausalConv3d(z_dim * 2, z_dim * 2, 1) + self.conv2 = CausalConv3d(z_dim, z_dim, 1) + self.decoder = Decoder3d( + dec_dim, z_dim, self.dim_mult, num_res_blocks, self.attn_scales, self.temporal_upsample, dropout + ) + + def encode(self, x, scale): + self.clear_cache() + x = _patchify(x, patch_size=2) + t = x.shape[2] + iter_ = 1 + (t - 1) // 4 + out = None + for i in range(iter_): + self._enc_conv_idx = [0] + if i == 0: + out = self.encoder(x[:, :, :1, :, :], feat_cache=self._enc_feat_map, feat_idx=self._enc_conv_idx) + else: + out_ = self.encoder( + x[:, :, 1 + 4 * (i - 1) : 1 + 4 * i, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx, + ) + out = torch.cat([out, out_], 2) + mu, log_var = self.conv1(out).chunk(2, dim=1) + if isinstance(scale[0], torch.Tensor): + mu = (mu - scale[0].view(1, self.z_dim, 1, 1, 1)) * scale[1].view(1, self.z_dim, 1, 1, 1) + else: + mu = (mu - scale[0]) * scale[1] + self.clear_cache() + return mu, log_var + + def decode(self, z, scale): + self.clear_cache() + if isinstance(scale[0], torch.Tensor): + z = z / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view(1, self.z_dim, 1, 1, 1) + else: + z = z / scale[1] + scale[0] + iter_ = z.shape[2] + x = self.conv2(z) + out = None + for i in range(iter_): + self._conv_idx = [0] + if i == 0: + out = self.decoder( + x[:, :, i : i + 1, :, :], feat_cache=self._feat_map, feat_idx=self._conv_idx, first_chunk=True + ) + else: + out_ = self.decoder(x[:, :, i : i + 1, :, :], feat_cache=self._feat_map, feat_idx=self._conv_idx) + out = torch.cat([out, out_], 2) + out = _unpatchify(out, patch_size=2) + self.clear_cache() + return out + + def clear_cache(self): + self._conv_num = _count_conv3d(self.decoder) + self._conv_idx = [0] + self._feat_map = [None] * self._conv_num + self._enc_conv_num = _count_conv3d(self.encoder) + self._enc_conv_idx = [0] + self._enc_feat_map = [None] * self._enc_conv_num + + +# ---------------------------------------------------------------------------- +# Wan2.2 VAE wrapper + BAGEL-surface adapter +# ---------------------------------------------------------------------------- + + +# Per-channel mean/std for the 48-channel Wan2.2 latent space (upstream constants). +_WAN22_LATENT_MEAN: tuple[float, ...] = ( + -0.2289, + -0.0052, + -0.1323, + -0.2339, + -0.2799, + 0.0174, + 0.1838, + 0.1557, + -0.1382, + 0.0542, + 0.2813, + 0.0891, + 0.1570, + -0.0098, + 0.0375, + -0.1825, + -0.2246, + -0.1207, + -0.0698, + 0.5109, + 0.2665, + -0.2108, + -0.2158, + 0.2502, + -0.2055, + -0.0322, + 0.1109, + 0.1567, + -0.0729, + 0.0899, + -0.2799, + -0.1230, + -0.0313, + -0.1649, + 0.0117, + 0.0723, + -0.2839, + -0.2083, + -0.0520, + 0.3748, + 0.0152, + 0.1957, + 0.1433, + -0.2944, + 0.3573, + -0.0548, + -0.1681, + -0.0667, +) +_WAN22_LATENT_STD: tuple[float, ...] = ( + 0.4765, + 1.0364, + 0.4514, + 1.1677, + 0.5313, + 0.4990, + 0.4818, + 0.5013, + 0.8158, + 1.0344, + 0.5894, + 1.0901, + 0.6885, + 0.6165, + 0.8454, + 0.4978, + 0.5759, + 0.3523, + 0.7135, + 0.6804, + 0.5833, + 1.4146, + 0.8986, + 0.5659, + 0.7069, + 0.5338, + 0.4889, + 0.4917, + 0.4069, + 0.4999, + 0.6866, + 0.4093, + 0.5709, + 0.6065, + 0.6415, + 0.4944, + 0.5726, + 1.2042, + 0.5458, + 1.6887, + 0.3971, + 1.0600, + 0.3943, + 0.5537, + 0.5444, + 0.4089, + 0.7468, + 0.7744, +) + + +class LanceWanVAE(nn.Module): + """Wan2.2 VAE wrapped for BAGEL's pipeline. + + Exposes BAGEL's image-VAE surface — ``encode(BCHW) -> BC_zHW`` and + ``decode(BC_zHW) -> BCHW`` — by treating each image as a 1-frame video clip. + A 5-D ``encode_video``/``decode_video`` path is also provided for the + ``Lance_3B_Video`` checkpoint. + + Construction is lazy: the heavy ``WanVAE_`` and ``Wan2.2_VAE.pth`` are not + materialized until first use. Once built, the inner module is registered as + a submodule so ``self.parameters()``, ``self.to(device)`` and ``vae_dtype = + next(vae.parameters()).dtype`` (used by BAGEL's decode path) all behave. + """ + + z_channels: int = 48 + downsample_spatial: int = 16 + downsample_temporal: int = 4 + + def __init__( + self, + vae_path: str, + dtype: torch.dtype = torch.bfloat16, + device: torch.device | str | None = None, + *, + lazy: bool = False, + ): + super().__init__() + self._vae_path = vae_path + self._dtype = dtype + self._device = torch.device(device) if device is not None else None + self._built = False + # Registered lazily once the .pth is loaded: + # self.model: WanVAE_ (set in _ensure_built) + if not lazy: + # BAGEL's decode path inspects ``next(vae.parameters()).dtype`` before + # the first encode/decode call, so we have to register the submodule + # eagerly when a real device is provided. ``lazy=True`` is reserved + # for config-only / no-GPU paths. + self._ensure_built() + + def _ensure_built(self) -> None: + if self._built: + return + device = self._device or ( + next(self.parameters(), torch.empty(0)).device + if any(True for _ in self.parameters()) + else torch.device("cpu") + ) + # Wan2.2 VAE config from upstream Wan2_2_VAE defaults: + # z_dim=48 (channels), c_dim=160 (encoder width), dec_dim=256 (decoder width), + # dim_mult=[1,2,4,4], temperal_downsample=[False,True,True]. + model = WanVAE_( + dim=160, + dec_dim=256, + z_dim=self.z_channels, + dim_mult=(1, 2, 4, 4), + num_res_blocks=2, + attn_scales=(), + temperal_downsample=(False, True, True), + dropout=0.0, + ) + logger.info("Loading Wan2.2 VAE state dict from %s", self._vae_path) + state = torch.load(self._vae_path, map_location="cpu", weights_only=True) + missing, unexpected = model.load_state_dict(state, strict=False) + if missing: + logger.warning("Wan2.2 VAE missing keys (%d): %s", len(missing), missing[:5]) + if unexpected: + logger.warning("Wan2.2 VAE unexpected keys (%d): %s", len(unexpected), unexpected[:5]) + model = model.to(device=device, dtype=self._dtype).eval() + model.requires_grad_(False) + # Register as submodule so .to() / .parameters() pick it up. + self.model = model + # Per-channel normalization tensors (matches upstream Wan2_2_VAE.scale = [mean, 1/std]). + mean = torch.tensor(_WAN22_LATENT_MEAN, dtype=self._dtype, device=device) + inv_std = torch.tensor([1.0 / s for s in _WAN22_LATENT_STD], dtype=self._dtype, device=device) + # Buffers, so .to(device) follows along. + self.register_buffer("_latent_mean", mean, persistent=False) + self.register_buffer("_latent_inv_std", inv_std, persistent=False) + self._built = True + + @torch.inference_mode() + def encode_video(self, video: torch.Tensor, *, use_sample: bool = True) -> torch.Tensor: + """Encode a 5-D clip ``[B, 3, T, H, W]`` -> latent ``[B, 48, t, h, w]``.""" + self._ensure_built() + video = video.to(self.model.encoder.conv1.weight.dtype) + mu, log_var = self.model.encode(video, [self._latent_mean, self._latent_inv_std]) + if use_sample: + std = torch.exp(0.5 * log_var) + mu = mu + std * torch.randn_like(std) + return mu + + @torch.inference_mode() + def decode_video(self, latent: torch.Tensor) -> torch.Tensor: + """Decode a 5-D latent ``[B, 48, t, h, w]`` -> video ``[B, 3, T, H, W]``.""" + self._ensure_built() + latent = latent.to(self.model.decoder.conv1.weight.dtype) + out = self.model.decode(latent, [self._latent_mean, self._latent_inv_std]) + return out.clamp_(-1.0, 1.0) + + # ----- BAGEL image-VAE surface (4-D) -------------------------------- # + + @torch.inference_mode() + def encode(self, padded_images: torch.Tensor) -> torch.Tensor: + """``[B, 3, H, W]`` -> ``[B, 48, H/16, W/16]`` (single-frame image path). + + Each image is wrapped as a 1-frame clip, encoded, and the temporal axis + is squeezed back out so the result matches BAGEL's iteration pattern. + """ + if padded_images.dim() == 4: + video = padded_images.unsqueeze(2) # B,C,1,H,W + squeeze_time = True # image path: caller iterates 3-D latents + elif padded_images.dim() == 5: + video = padded_images + # video path: caller expects 4-D latents (C,T,H,W), even when T==1 + # (e.g. video2video with a 1-frame reference clip). Squeezing here + # would drop the temporal axis and break the downstream indexing + # ``latent[:, :t_lat, ...]`` in ``forward_cache_update_vae``. + squeeze_time = False + else: + raise ValueError(f"LanceWanVAE.encode expects 4-D BCHW or 5-D BCTHW, got {tuple(padded_images.shape)}") + latent = self.encode_video(video, use_sample=True) + # B,48,t,h,w -> B,48,h,w when t == 1 (image path only) + if squeeze_time and latent.shape[2] == 1: + return latent.squeeze(2) + return latent + + @torch.inference_mode() + def decode(self, latent: torch.Tensor) -> torch.Tensor: + """``[B, 48, h, w]`` -> ``[B, 3, H, W]`` (single-frame image path).""" + if latent.dim() == 4: + latent = latent.unsqueeze(2) # B,48,1,h,w + elif latent.dim() != 5: + raise ValueError(f"LanceWanVAE.decode expects 4-D BCHW or 5-D BCTHW, got {tuple(latent.shape)}") + video = self.decode_video(latent) # B,3,T,H,W + if video.shape[2] == 1: + return video.squeeze(2) + return video + + +def build_wan22_vae(vae_path: str, dtype: torch.dtype = torch.bfloat16, device=None) -> LanceWanVAE: + """Convenience factory: lazy-construct a :class:`LanceWanVAE` adapter.""" + return LanceWanVAE(vae_path=vae_path, dtype=dtype, device=device) + + +__all__ = ["LanceWanVAE", "WanVAE_", "build_wan22_vae"] diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index a381640b5..fe9ed8fcf 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -141,6 +141,11 @@ _DIFFUSION_MODELS = { "pipeline_bagel", "BagelPipeline", ), + "LancePipeline": ( + "lance", + "pipeline_lance", + "LancePipeline", + ), "MingImagePipeline": ( "ming_flash_omni", "pipeline_ming_imagegen", @@ -471,6 +476,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = { "WanI2VDMD2Pipeline": "get_wan22_i2v_post_process_func", "LongCatImagePipeline": "get_longcat_image_post_process_func", "BagelPipeline": "get_bagel_post_process_func", + "LancePipeline": "get_lance_post_process_func", "MingImagePipeline": "get_ming_image_post_process_func", "InternVLAA1Pipeline": "get_internvla_a1_post_process_func", "LongCatImageEditPipeline": "get_longcat_image_post_process_func", diff --git a/vllm_omni/model_executor/models/lance/__init__.py b/vllm_omni/model_executor/models/lance/__init__.py new file mode 100644 index 000000000..2ecd5df1f --- /dev/null +++ b/vllm_omni/model_executor/models/lance/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Lance (ByteDance) — vLLM-Omni model_executor package. + +Single-stage topology: a self-contained diffusion stage (``LancePipeline`` +defined in ``vllm_omni.diffusion.models.lance``) that owns the Qwen2-MoT LLM, +the Qwen2.5-VL ViT, the Wan2.2 VAE and the tokenizer — directly analogous to +``bagel_single_stage``. The ``PipelineConfig`` lives in :mod:`.pipeline`. +""" diff --git a/vllm_omni/model_executor/models/lance/pipeline.py b/vllm_omni/model_executor/models/lance/pipeline.py new file mode 100644 index 000000000..777f8942d --- /dev/null +++ b/vllm_omni/model_executor/models/lance/pipeline.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Lance pipeline topology. + +Single-stage DiT — self-contained diffusion stage that handles all modalities +(text2img, image_edit, x2t_image, video) internally via its own Qwen2-MoT +LLM, Qwen2.5-VL ViT, Wan2.2 VAE, and tokenizer. Mirrors ``bagel_single_stage``. +""" + +from vllm_omni.config.stage_config import ( + PipelineConfig, + StageExecutionType, + StagePipelineConfig, +) + +LANCE_PIPELINE = PipelineConfig( + model_type="lance", + model_arch="LancePipeline", + hf_architectures=(), + stages=( + StagePipelineConfig( + stage_id=0, + model_stage="dit", + execution_type=StageExecutionType.DIFFUSION, + input_sources=(), + final_output=True, + final_output_type="image", + ), + ), +)