diff --git a/.gitignore b/.gitignore index 378de441c..40a6916b5 100644 --- a/.gitignore +++ b/.gitignore @@ -270,3 +270,6 @@ vllm_omni/_version.py .worktrees/ # CI overlay yamls materialized from tests/utils.py:_CI_OVERLAYS at test time tests/.ci_generated/ + +# Local Lance precision-matching dump scripts (not part of the repo) +lance_compare/ diff --git a/LANCE_PR.md b/LANCE_PR.md deleted file mode 100644 index d42ccce20..000000000 --- a/LANCE_PR.md +++ /dev/null @@ -1,76 +0,0 @@ -# [New Model] Lance (ByteDance) — #3697 - -Adds Lance (`bytedance-research/Lance`), a 3B unified AR+diffusion -multimodal model on a Qwen2.5-VL backbone. - -## Why this is BAGEL, not WAN - -The issue thread suggested remapping against `WanTransformer3DModel`. The -released `Lance_3B` checkpoint is instead **BAGEL Mixture-of-Transformers**: -every layer carries `mlp_moe_gen` / `q_proj_moe_gen` / -`input_layernorm_moe_gen` twins plus `q_norm`/`k_norm`, with top-level -`vae2llm` / `llm2vae` / `time_embedder` / `latent_pos_embed` connectors — -identical naming to vLLM-Omni's BAGEL. The only WAN element is the VAE. -Lance therefore reuses BAGEL's transformer core verbatim and specializes -only the LLM (Qwen2.5-VL-MoT), ViT (Qwen2.5-VL vision), VAE (Wan2.2) and -checkpoint layout. - -## Scope and status - -All six single-stage Lance modalities, verified end-to-end on 1× B300: - -| Modality | Status | -|----------|--------| -| `t2i` (text→image) | working — 1024×1024, 0 missing keys | -| `t2v` (text→video) | working — photorealistic frames with motion; 768×768 trained bucket | -| `image_edit` (img2img) | working — Lance-native VAE prefill scatters Wan2.2 latents into the LLM query sequence | -| `video_edit` (video2video) | working — multi-frame VAE prefill + 3-D mRoPE; needs the `Lance_3B_Video` checkpoint | -| `x2t_image` (image understanding) | working — Qwen2.5-VL ViT prefill + 3-D mRoPE | -| `x2t_video` (video understanding) | working — Qwen2-VL video processor + 3-D `video_grid_thw` | - -`rope_scaling = {"type": "mrope", "mrope_section": [16, 24, 24]}` is wired -through `BagelRotaryEmbedding`: t2i uses scalar position ids -(BAGEL-equivalent); x2t / video / edit thread per-axis 3-D position ids. -Geometry constants (`latent_patch_size_spatial = 1`, `max_latent_size = 64`, -untied `lm_head`, Wan2.2 VAE `z=48 /16 spatial /4 temporal`) were locked in -by inspecting the released checkpoint directly. - -The two-stage AR+DiT topology is the only deferred path (needs -`LanceConfig` / `LanceProcessor` in the `vllm` package — a separate -human-authored PR per vLLM's contribution policy). - -## Test plan - -Offline (`examples/offline_inference/lance/end2end.py`), e.g. t2i: - -```bash -python examples/offline_inference/lance/end2end.py \ - --model bytedance-research/Lance \ - --prompts "a corgi astronaut on the moon, cinematic" \ - --steps 30 --cfg-text-scale 4.0 --timestep-shift 3.5 --output ./out -``` - -`--modality {text2video,img2img,video2video,img2text,video2text}` selects -the other paths; video paths use `--model bytedance-research/Lance/Lance_3B_Video`. - -Online (OpenAI-compatible, `examples/online_serving/lance/`): - -```bash -bash examples/online_serving/lance/run_server.sh # vllm serve … --omni -python examples/online_serving/lance/openai_chat_client.py \ - --prompt "A cute corgi astronaut on the moon, cinematic" \ - --modality text2img --output corgi.png -``` - -E2E coverage: `tests/e2e/online_serving/test_lance.py`. Tested on 1× -NVIDIA B300, `torch 2.11.0+cu130`, stock attention backend. - -## Follow-ups - -1. **Two-stage AR + DiT** — register `LanceConfig` / `LanceProcessor` in - the `vllm` package; separate human-authored PR per vLLM policy. -2. **`video_edit` quality polish** — output is more abstract than `t2v` - at the same resolution (position-id offset between the VAE-prefill and - gen-latent blocks). Functionally correct end-to-end. - -Closes #3697. diff --git a/examples/offline_inference/lance/README.md b/examples/offline_inference/lance/README.md index 0697c50c7..d029abdea 100644 --- a/examples/offline_inference/lance/README.md +++ b/examples/offline_inference/lance/README.md @@ -23,6 +23,7 @@ python examples/offline_inference/lance/end2end.py \ --model bytedance-research/Lance \ --prompts "a corgi astronaut on the moon, cinematic" \ --steps 30 --cfg-text-scale 4.0 --timestep-shift 3.5 \ + --height 1024 --width 1024 \ --output ./out # Text-to-video (uses the Lance_3B_Video subfolder; see ``--modality`` @@ -45,6 +46,7 @@ The HF repo bundles everything (`Lance_3B/`, `Lance_3B_Video/`, ## Defaults Matches upstream `inference_lance.sh`: 30 denoising steps, timestep-shift 3.5, -text CFG 4.0, seed 42, 768×768. For the understanding paths (`img2text` / -`video2text`), sampling is enabled by default at `--text-temperature 0.8` -because Lance's greedy decoder emits an immediate EOS for many prompts. +text CFG 4.0, seed 42, 1024×1024 (override with `--height` / `--width`). For +the understanding paths (`img2text` / `video2text`), sampling is enabled by +default at `--text-temperature 0.8` because Lance's greedy decoder emits an +immediate EOS for many prompts. diff --git a/examples/offline_inference/lance/end2end.py b/examples/offline_inference/lance/end2end.py index 2f0a94dea..344699fc8 100644 --- a/examples/offline_inference/lance/end2end.py +++ b/examples/offline_inference/lance/end2end.py @@ -41,6 +41,8 @@ def parse_args(): p.add_argument("--num-frames", type=int, default=25, help="Number of RGB frames for text2video (max 121).") p.add_argument("--video-height", type=int, default=480, help="Video frame height.") p.add_argument("--video-width", type=int, default=768, help="Video frame width.") + p.add_argument("--height", type=int, default=None, help="Image height (t2i). Default = max_hw (1024).") + p.add_argument("--width", type=int, default=None, help="Image width (t2i). Default = max_hw (1024).") p.add_argument("--fps", type=int, default=8, help="Output video FPS when saving MP4.") p.add_argument("--output", type=str, default=".", help="Output directory.") p.add_argument("--steps", type=int, default=30, help="Denoising steps (Lance default 30).") @@ -207,6 +209,10 @@ def main(): diffusion_params.num_inference_steps = args.steps # type: ignore if args.seed is not None: diffusion_params.seed = args.seed # type: ignore + if args.height is not None: + diffusion_params.height = args.height # type: ignore + if args.width is not None: + diffusion_params.width = args.width # type: ignore extra = getattr(diffusion_params, "extra_args", {}) or {} extra["cfg_text_scale"] = args.cfg_text_scale extra["timestep_shift"] = args.timestep_shift diff --git a/recipes/ByteDance/Lance.md b/recipes/ByteDance/Lance.md new file mode 100644 index 000000000..ea5700bdc --- /dev/null +++ b/recipes/ByteDance/Lance.md @@ -0,0 +1,177 @@ +# Lance + +> Unified autoregressive + diffusion multimodal (text / image / video) + +## Summary + +- Vendor: ByteDance +- Model: [`bytedance-research/Lance`](https://huggingface.co/bytedance-research/Lance) (`Lance_3B`, `Lance_3B_Video`) +- Task: text2img, text2video, img2img (image edit), video2video (video edit), + img2text (image understanding), video2text (video understanding) +- Mode: Offline inference, Online serving (OpenAI-compatible API) +- Maintainer: Community + +## When to use this recipe + +Use this recipe to run Lance — a 3B unified AR + diffusion multimodal model +on a Qwen2.5-VL backbone — via vLLM-Omni. Lance is **BAGEL-lineage**: the +released checkpoint uses the same `*_moe_gen` Mixture-of-Transformers weight +layout as BAGEL, so vLLM-Omni reuses BAGEL's transformer core and specializes +only the ViT (Qwen2.5-VL vision), the VAE (Wan2.2) and the checkpoint layout. + +All six single-stage modalities are supported on the `Lance_3B` checkpoint; +`text2video` and `video2video` additionally need `Lance_3B_Video` for the 3-D +`latent_pos_embed` table. The two-stage AR + DiT topology is deferred and +needs `LanceConfig` / `LanceProcessor` registered in the `vllm` package (a +separate upstream PR). + +## References + +- Offline example: + [`examples/offline_inference/lance/end2end.py`](../../examples/offline_inference/lance/end2end.py) +- Online serving: + [`examples/online_serving/lance/`](../../examples/online_serving/lance/) +- Deploy config: + [`vllm_omni/deploy/lance.yaml`](../../vllm_omni/deploy/lance.yaml) +- E2E tests: + [`tests/e2e/online_serving/test_lance.py`](../../tests/e2e/online_serving/test_lance.py) +- HuggingFace model page: + [bytedance-research/Lance](https://huggingface.co/bytedance-research/Lance) +- PR: [vllm-project/vllm-omni#3710](https://github.com/vllm-project/vllm-omni/pull/3710) + (closes [#3697](https://github.com/vllm-project/vllm-omni/issues/3697)) + +## Hardware Support + +## GPU + +### 1x NVIDIA B300 / A100 80GB + +#### Environment + +- OS: Linux +- Python: 3.12 +- Driver / runtime: CUDA ≥ 12.4 +- vLLM-Omni version: 0.18.x.dev + +#### Command — text-to-image (default) + +```bash +python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance \ + --prompts "a corgi astronaut on the moon, cinematic" \ + --steps 30 --cfg-text-scale 4.0 --timestep-shift 3.5 \ + --height 1024 --width 1024 \ + --seed 42 --output ./out +``` + +Defaults match upstream `inference_lance.sh`: 30 denoising steps, +`timestep-shift 3.5`, text CFG 4.0, seed 42, 1024×1024. + +#### Command — image edit (img2img) + +```bash +python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance --modality img2img \ + --image-path /path/to/input.png \ + --prompts "Convert this into a vibrant cartoon-style illustration" \ + --steps 30 --cfg-text-scale 4.0 --timestep-shift 3.5 \ + --output ./out +``` + +The Lance-native VAE prefill scatters Wan2.2 latents into the LLM query +sequence; no separate image encoder is needed. + +#### Command — text-to-video + +```bash +python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance/Lance_3B_Video --modality text2video \ + --num-frames 25 --video-height 480 --video-width 768 \ + --prompts "a cat playing piano, cinematic" \ + --steps 30 --fps 8 --output ./out +``` + +Use the `Lance_3B_Video` subfolder for any video path so the 3-D +`latent_pos_embed` table is loaded; image / understanding paths can point at +the top-level repo and resolve the right sub-checkpoint automatically. + +#### Command — image / video understanding + +```bash +# Image → text (caption / VQA) +python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance --modality img2text \ + --image-path /path/to/photo.jpg \ + --prompts "Describe this image in detail." \ + --do-sample --text-temperature 0.8 + +# Video → text +python examples/offline_inference/lance/end2end.py \ + --model bytedance-research/Lance --modality video2text \ + --video-path /path/to/clip.mp4 \ + --prompts "What is happening in this video?" +``` + +Sampling is enabled by default at `--text-temperature 0.8` for the +understanding paths because Lance's greedy decoder emits an immediate EOS for +many prompts. + +#### Verification + +```bash +pytest -s -v tests/e2e/online_serving/test_lance.py +``` + +#### Notes + +- BF16 footprint: ~7 GB LLM + Qwen2.5-VL ViT + Wan2.2 VAE; comfortably fits + on a single 16 GB+ GPU for `Lance_3B`. +- `rope_scaling = {"type": "mrope", "mrope_section": [16, 24, 24]}` is wired + through `BagelRotaryEmbedding`. `text2img` uses scalar position ids + (BAGEL-equivalent); `img2text` / `video` / edit paths thread per-axis 3-D + position ids. +- `video_edit` quality is more abstract than `text2video` at the same + resolution (known position-id offset between VAE-prefill and gen-latent + blocks). Functionally correct end-to-end. + +## Online Serving + +Lance supports all single-stage modalities via the OpenAI-compatible +`/v1/chat/completions` API. + +### Launch + +```bash +bash examples/online_serving/lance/run_server.sh +# or, with overrides +MODEL=bytedance-research/Lance \ +DEPLOY_CONFIG=vllm_omni/deploy/lance.yaml \ +PORT=8091 \ + bash examples/online_serving/lance/run_server.sh +``` + +For `text2video` / `video2video`, set +`MODEL=bytedance-research/Lance/Lance_3B_Video`. + +### Send requests + +```bash +# Text-to-image +python examples/online_serving/lance/openai_chat_client.py \ + --prompt "A cute corgi astronaut on the moon, cinematic" \ + --modality text2img --output corgi.png + +# Image edit +python examples/online_serving/lance/openai_chat_client.py \ + --prompt "Convert this into a vibrant cartoon-style illustration" \ + --modality img2img --image-url path/to/photo.png \ + --output edited.png + +# Image understanding +python examples/online_serving/lance/openai_chat_client.py \ + --prompt "Describe this image" \ + --modality img2text --image-url photo.jpg +``` + +The client is shared with BAGEL — same OpenAI message format, same +`modalities` / `num_inference_steps` / `seed` / `height` / `width` knobs. diff --git a/recipes/README.md b/recipes/README.md index 09b1aeaee..d3cde3063 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -29,6 +29,7 @@ recipes/ |--------|------|----------| | [`audiox/AudioX.md`](./audiox/AudioX.md) | Offline + online unified text/video→audio diffusion | 1x L4 24GB | | [`Baidu/ERNIE-Image.md`](./Baidu/ERNIE-Image.md) | Text-to-image online serving (ERNIE-Image 8B) | 1x or 2x RTX 4090 24GB | +| [`ByteDance/Lance.md`](./ByteDance/Lance.md) | Unified AR+diffusion: text/img/video gen + understanding (Lance 3B) | 1x B300 / A100 80GB | | [`fishaudio/Fish-Speech-S2-Pro.md`](./fishaudio/Fish-Speech-S2-Pro.md) | Online serving for TTS | 1x A800 80GB | | [`inclusionAI/Ming-flash-omni-2.0.md`](./inclusionAI/Ming-flash-omni-2.0.md) | Online serving for multimodal chat + standalone TTS | 4x H100 / 1x H100 80GB | | [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB | diff --git a/vllm_omni/diffusion/models/bagel/bagel_transformer.py b/vllm_omni/diffusion/models/bagel/bagel_transformer.py index 22e7506e7..7ed65a1af 100644 --- a/vllm_omni/diffusion/models/bagel/bagel_transformer.py +++ b/vllm_omni/diffusion/models/bagel/bagel_transformer.py @@ -1822,7 +1822,11 @@ class Bagel(nn.Module): ): x_t = packed_init_noises - timesteps = torch.linspace(1, 0, num_timesteps, device=x_t.device) + # Use num_timesteps + 1 sample points so we get `num_timesteps` denoise + # steps after dropping the terminal t=0 (which has no dt). Upstream + # Lance / BAGEL both use this convention; without the +1 we silently + # run one fewer denoise iteration than the user asked for. + timesteps = torch.linspace(1, 0, num_timesteps + 1, device=x_t.device) timesteps = timestep_shift * timesteps / (1 + (timestep_shift - 1) * timesteps) dts = timesteps[:-1] - timesteps[1:] timesteps = timesteps[:-1] diff --git a/vllm_omni/diffusion/models/bagel/pipeline_bagel.py b/vllm_omni/diffusion/models/bagel/pipeline_bagel.py index 2363a690a..a9bd897a9 100644 --- a/vllm_omni/diffusion/models/bagel/pipeline_bagel.py +++ b/vllm_omni/diffusion/models/bagel/pipeline_bagel.py @@ -355,7 +355,7 @@ class BagelPipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipelineProf gen_params = BagelGenParams( num_timesteps=int(req.sampling_params.num_inference_steps or 50), - timestep_shift=3.0, + timestep_shift=float(extra_args.get("timestep_shift", 3.0)), cfg_text_scale=cfg_text_scale, cfg_img_scale=cfg_img_scale, cfg_interval=cfg_interval, diff --git a/vllm_omni/diffusion/models/lance/lance_transformer.py b/vllm_omni/diffusion/models/lance/lance_transformer.py index 2000508f6..bdc495055 100644 --- a/vllm_omni/diffusion/models/lance/lance_transformer.py +++ b/vllm_omni/diffusion/models/lance/lance_transformer.py @@ -75,6 +75,40 @@ __all__ = [ LANCE_TOKENS_PER_SECOND = 2 LANCE_SECONDS_PER_GRID = 1.0 +# Upstream Lance ViT preprocessing (see lance-upstream +# ``data/datasets_custom/validation_dataset.py`` + ``data/video/transforms/``): +# ``VideoTransform(resolution=resolution_vit, mode='bucket', +# divisible_crop_size=28, stride_spatial=16, aspect_ratios=[...])``. +# For ``image_768res`` (Lance's default for 768px image gen), resolution_vit=672. +# Resizes to the nearest aspect-ratio bucket (max_area=672², stride=16), then +# center-crops to dims divisible by 28 (=patch_size * spatial_merge_size). +# Standard Qwen2VLImageProcessor's smart-resize is NOT used by upstream — the +# image enters the ViT at the bucket-cropped dimensions. +LANCE_VIT_BUCKET_RESOLUTION = 672 +LANCE_VIT_BUCKET_STRIDE = 16 +LANCE_VIT_DIVISIBLE_CROP = 28 # patch_size (14) * spatial_merge_size (2) +LANCE_VIT_ASPECT_RATIOS = ("21:9", "16:9", "4:3", "1:1", "3:4", "9:16") + +# Video-specific bucket parameters (resolution=video_480p — Lance's default for +# t2v / video_edit at 12 fps). See lance-upstream +# ``data/datasets_custom/validation_dataset.py`` lines 128-156: +# VAE side: resolution_vae=640, divisible_crop_size=16, mean=0.5, std=0.5 +# ViT side: resolution_vit=616, divisible_crop_size=28, +# mean=Qwen2.5-VL CLIP-norm, std=Qwen2.5-VL CLIP-norm +LANCE_VIDEO_VAE_RESOLUTION = 640 +LANCE_VIDEO_VAE_DIVISIBLE_CROP = 16 +LANCE_VIDEO_VIT_RESOLUTION = 616 +LANCE_VIDEO_VIT_DIVISIBLE_CROP = 28 +LANCE_VIDEO_BUCKET_STRIDE = 16 +LANCE_VIDEO_TEMPORAL_STRIDE = 4 # Wan2.2 VAE temporal downsample +LANCE_VIDEO_SAMPLE_FPS = 12 # upstream MultiClipsFrameSampler default +LANCE_VIDEO_MAX_DURATION = 6.0 # upstream max_duration (seconds) +LANCE_VIT_PATCH_SIZE = 14 # Qwen2.5-VL ViT spatial patch +LANCE_VIT_TEMPORAL_PATCH_SIZE = 2 # Qwen2.5-VL ViT temporal patch +LANCE_VIT_SPATIAL_MERGE = 2 # Qwen2.5-VL spatial merge factor +LANCE_VIT_NORM_MEAN = (0.48145466, 0.4578275, 0.40821073) +LANCE_VIT_NORM_STD = (0.26862954, 0.26130258, 0.27577711) + def get_3d_sincos_pos_embed_from_grid(embed_dim: int, grid: np.ndarray) -> np.ndarray: """3-D sin-cos positional embedding (t, h, w), matching the upstream Lance @@ -244,10 +278,356 @@ class LanceBagel(Bagel): ``forward_cache_update_vae`` — is reused unchanged. """ + @staticmethod + def _lance_compute_vit_buckets(): + """Replicate upstream Lance's ``BucketResize.init_buckets`` for the ViT path. + + Returns a list of ``((bucket_h, bucket_w), bucket_ratio)`` pairs in the + same order as :data:`LANCE_VIT_ASPECT_RATIOS`. Stays in sync with + upstream lance ``data/video/transforms/bucket_resize.py``. + """ + import math + + max_area = LANCE_VIT_BUCKET_RESOLUTION * LANCE_VIT_BUCKET_RESOLUTION + stride = LANCE_VIT_BUCKET_STRIDE + buckets: list[tuple[tuple[int, int], float]] = [] + for name in LANCE_VIT_ASPECT_RATIOS: + w_s, h_s = (int(v) for v in name.split(":")) + aspect = w_s / h_s + # Option 1: solve for width first. + rw1 = math.sqrt(max_area * aspect) + bw1 = round(rw1 / stride) * stride + bh1 = round(bw1 / aspect / stride) * stride + br1 = bw1 / bh1 + # Option 2: solve for height first. + rh2 = math.sqrt(max_area / aspect) + bh2 = round(rh2 / stride) * stride + bw2 = round(bh2 * aspect / stride) * stride + br2 = bw2 / bh2 + if abs(br1 - aspect) < abs(br2 - aspect): + bh, bw = bh1, bw1 + elif abs(br1 - aspect) > abs(br2 - aspect): + bh, bw = bh2, bw2 + else: + area1 = bh1 * bw1 + area2 = bh2 * bw2 + if abs(area1 - max_area) <= abs(area2 - max_area): + bh, bw = bh1, bw1 + else: + bh, bw = bh2, bw2 + buckets.append(((bh, bw), bw / bh)) + return buckets + + @staticmethod + def _lance_bucket_resize_pil(image): + """Apply upstream Lance's ViT preprocessing to a PIL image. + + Pipeline (matches lance-upstream's ``VideoTransform`` for the ViT path): + + 1. ``BucketResize``: pick the aspect-ratio bucket nearest to the input + and resize via ``RandomResizedCrop(scale=(1,1), ratio=fixed)``, + which for fixed ratio is a deterministic center-crop + resize. + 2. ``DivisibleCrop(28)``: center-crop to dims divisible by + ``patch_size * spatial_merge_size`` so the resulting ViT grid is + clean. + + Returns the post-pipeline PIL image (still RGB, just resized). Doing + this BEFORE the Qwen2VLImageProcessor means smart-resize is a no-op + and the ViT sees exactly upstream Lance's input. + """ + import numpy as _np + from torchvision.transforms import InterpolationMode, RandomResizedCrop + + if image.mode != "RGB": + image = image.convert("RGB") + w, h = image.size + img_ratio = w / h + + buckets = LanceBagel._lance_compute_vit_buckets() + # nearest bucket by absolute ratio difference + diffs = _np.array([abs(img_ratio - br) for _, br in buckets]) + bh, bw = buckets[int(diffs.argmin())][0] + bratio = bw / bh + + # RandomResizedCrop(scale=(1,1), ratio=(bratio, bratio)) is deterministic: + # picks the largest sub-image of the target aspect, then resizes to + # (bh, bw). We replicate this directly to avoid the torchvision + # transform's random-state dependency and to keep the result a PIL. + # IMPORTANT: upstream Lance wires BucketResize through ``NaResize`` + # which defaults to BICUBIC interpolation (NOT LANCZOS — BucketResize's + # own default is LANCZOS but ``NaResize`` overrides it). Using + # LANCZOS here gave byte-different ViT input pixels (max_diff~0.13) + # which fed into the noise-query attention as wrong K/V — symptom: + # vllm-omni preserved the input image's hat when upstream removed it. + rrc = RandomResizedCrop( + size=(bh, bw), + scale=(1.0, 1.0), + ratio=(bratio, bratio), + interpolation=InterpolationMode.BICUBIC, + ) + out = rrc(image) + # DivisibleCrop(28): center-crop to dims divisible by 28. + ow, oh = out.size + cropped_w = ow - (ow % LANCE_VIT_DIVISIBLE_CROP) + cropped_h = oh - (oh % LANCE_VIT_DIVISIBLE_CROP) + if cropped_w != ow or cropped_h != oh: + left = (ow - cropped_w) // 2 + top = (oh - cropped_h) // 2 + out = out.crop((left, top, left + cropped_w, top + cropped_h)) + return out + + # ------------------------------------------------------------------ # + # Video preprocessing — frame sampler + bucket resize for VAE / ViT + # ------------------------------------------------------------------ # + @staticmethod + def _lance_compute_video_buckets(resolution: int, stride: int): + """Generic version of :meth:`_lance_compute_vit_buckets` for arbitrary + ``resolution`` / ``stride`` (the video VAE uses 640/16, the video ViT + uses 616/16, both with the same 6 aspect-ratio table).""" + import math + + max_area = resolution * resolution + buckets: list[tuple[tuple[int, int], float]] = [] + for name in LANCE_VIT_ASPECT_RATIOS: + w_s, h_s = (int(v) for v in name.split(":")) + aspect = w_s / h_s + rw1 = math.sqrt(max_area * aspect) + bw1 = round(rw1 / stride) * stride + bh1 = round(bw1 / aspect / stride) * stride + br1 = bw1 / bh1 + rh2 = math.sqrt(max_area / aspect) + bh2 = round(rh2 / stride) * stride + bw2 = round(bh2 * aspect / stride) * stride + br2 = bw2 / bh2 + if abs(br1 - aspect) < abs(br2 - aspect): + bh, bw = bh1, bw1 + elif abs(br1 - aspect) > abs(br2 - aspect): + bh, bw = bh2, bw2 + else: + area1 = bh1 * bw1 + area2 = bh2 * bw2 + bh, bw = (bh1, bw1) if abs(area1 - max_area) <= abs(area2 - max_area) else (bh2, bw2) + buckets.append(((bh, bw), bw / bh)) + return buckets + + @staticmethod + def _lance_sample_frame_indices(num_input_frames: int, origin_fps: float) -> list[int]: + """Replicate upstream's ``MultiClipsFrameSampler`` (single-clip path). + + Upstream config (validation_dataset.py:110): + temporal=4, sample_fps=12, truncate=False, max_duration=6.0, + length_type="kn+1", assert_seconds=False. + + For a single clip covering the full input video this reduces to: + + duration = min(num_input_frames / origin_fps, max_duration) + n_frames = round(duration * sample_fps) + # round to k*temporal + 1 (or -temporal+1 if already a multiple) + if n_frames % temporal != 0: + n_frames = (n_frames // temporal) * temporal + 1 + else: + n_frames = (n_frames // temporal) * temporal + 1 - temporal + indices = np.linspace(0, num_input_frames - 1, n_frames, dtype=int) + + Verified against upstream for car (93 frames @ 24fps → 45 sampled) and + woman (121 frames @ 24fps → 57 sampled). + """ + import numpy as _np + + duration = num_input_frames / origin_fps + duration = min(duration, LANCE_VIDEO_MAX_DURATION) + n_frames = int(round(duration * LANCE_VIDEO_SAMPLE_FPS)) + t = LANCE_VIDEO_TEMPORAL_STRIDE + if n_frames % t != 0: + n_frames = (n_frames // t) * t + 1 + else: + n_frames = (n_frames // t) * t + 1 - t + n_frames = max(n_frames, 1) + return _np.linspace(0, num_input_frames - 1, n_frames, dtype=int).tolist() + + @staticmethod + def _lance_bucket_resize_video(frames_thwc, resolution: int, stride: int, divisible_crop: int, mean, std): + """Apply upstream Lance's VideoTransform pipeline to a (T,H,W,C) uint8 + ndarray. Returns ``(C, T, H', W')`` float tensor. + + Pipeline (mirrors lance-upstream's ``data/transforms.py::VideoTransform``): + + 1. BucketResize per PIL frame: RandomResizedCrop(scale=(1,1), + ratio=fixed-bucket-ratio, BICUBIC) → resized PIL → to_tensor. + Per-frame PIL path matches upstream's + ``BucketResize.__call__(list[PIL.Image])`` exactly — using a + batched tensor RRC instead drifted pixel values by ~0.13 + (in [0,1] range) because torchvision's tensor-BICUBIC kernel + differs subtly from PIL's BICUBIC kernel. + 2. DivisibleCrop(divisible_crop) — center-crop to dims divisible + by patch_size * spatial_merge_size (28) for ViT, or by VAE + downsample (16) for VAE. + 3. Normalize(mean, std) — VAE uses 0.5/0.5; ViT uses Qwen CLIP. + 4. Rearrange "t c h w -> c t h w" + """ + import numpy as _np + from PIL import Image as _Image + from torchvision.transforms import InterpolationMode, RandomResizedCrop + from torchvision.transforms.functional import center_crop as _ccrop + from torchvision.transforms.functional import to_tensor as _to_tensor + + if not isinstance(frames_thwc, _np.ndarray): + raise ValueError(f"Expected (T,H,W,C) uint8 ndarray; got {type(frames_thwc)}") + if frames_thwc.dtype != _np.uint8: + frames_thwc = frames_thwc.astype(_np.uint8) + if frames_thwc.ndim != 4 or frames_thwc.shape[-1] != 3: + raise ValueError(f"Expected (T,H,W,3) uint8; got {frames_thwc.shape}") + + T_in, H_in, W_in, _ = frames_thwc.shape + aspect = W_in / H_in + buckets = LanceBagel._lance_compute_video_buckets(resolution, stride) + diffs = _np.array([abs(aspect - br) for _, br in buckets]) + bh, bw = buckets[int(diffs.argmin())][0] + bratio = bw / bh + + rrc = RandomResizedCrop( + size=(bh, bw), + scale=(1.0, 1.0), + ratio=(bratio, bratio), + interpolation=InterpolationMode.BICUBIC, + ) + # Per-frame PIL → RRC → to_tensor → stack — matches upstream's + # ``BucketResize.__call__`` for ``list[PIL.Image]`` exactly. + per_frame = [] + for t in range(T_in): + img = _Image.fromarray(frames_thwc[t]) + img = rrc(img) + per_frame.append(_to_tensor(img)) # (C, bh, bw) float [0,1] + frames = torch.stack(per_frame, dim=0) # (T, C, bh, bw) + + # DivisibleCrop(divisible_crop) — center-crop along H/W. + _, C, Hb, Wb = frames.shape + Hc = Hb - (Hb % divisible_crop) + Wc = Wb - (Wb % divisible_crop) + if Hc != Hb or Wc != Wb: + frames = _ccrop(frames, [Hc, Wc]) + + # Normalize. Accept scalar or per-channel mean/std. + if isinstance(mean, (int, float)): + mean_t = torch.tensor([float(mean)] * C, dtype=frames.dtype).view(1, C, 1, 1) + else: + mean_t = torch.tensor(list(mean), dtype=frames.dtype).view(1, C, 1, 1) + if isinstance(std, (int, float)): + std_t = torch.tensor([float(std)] * C, dtype=frames.dtype).view(1, C, 1, 1) + else: + std_t = torch.tensor(list(std), dtype=frames.dtype).view(1, C, 1, 1) + frames = (frames - mean_t) / std_t + + # Rearrange "t c h w -> c t h w" + frames = frames.permute(1, 0, 2, 3).contiguous() + return frames + + @staticmethod + def _lance_patchify_vit_video(video_cthw: torch.Tensor): + """Replicate upstream ``patchify_video_with_merge``: (C, T, H, W) → + (num_patches, patch_dim). ``patch_dim = C * tp * p² = 3 * 2 * 14² = 1176``. + + Returns ``(packed_pixels, grid_thw)`` where ``grid_thw`` is the pre-merge + grid ``(T // tp, H // p, W // p)`` — same layout the Qwen2.5-VL ViT + expects (matches ``Qwen2VLImageProcessor.video_processor`` output up to + the actual pixel preprocessing). + """ + p = LANCE_VIT_PATCH_SIZE + tp = LANCE_VIT_TEMPORAL_PATCH_SIZE + ms = LANCE_VIT_SPATIAL_MERGE + # (C, T, H, W) -> (T, C, H, W) + video = video_cthw.permute(1, 0, 2, 3).contiguous() + T, C, H, W = video.shape + if T % tp != 0: + raise ValueError(f"ViT T={T} must be divisible by temporal_patch={tp}") + if H % p != 0 or W % p != 0: + raise ValueError(f"ViT H,W=({H},{W}) must be divisible by patch={p}") + gt, gh, gw = T // tp, H // p, W // p + if gh % ms != 0 or gw % ms != 0: + raise ValueError(f"ViT grid ({gh},{gw}) must be divisible by merge={ms}") + video = video.reshape(gt, tp, C, gh // ms, ms, p, gw // ms, ms, p) + # permute order matches upstream: (0,3,6,4,7,2,1,5,8) + video = video.permute(0, 3, 6, 4, 7, 2, 1, 5, 8) + patches = video.reshape(gt * gh * gw, C * tp * p * p) + grid_thw = torch.tensor([[gt, gh, gw]], dtype=torch.long) + return patches, grid_thw + + @classmethod + def _lance_video_preprocess(cls, frames_thwc, origin_fps: float): + """Full upstream-compatible video preprocessing for video_edit. + + Pipeline (mirrors lance-upstream ``validation_dataset.py`` for + ``--resolution video_480p``): + + 1. Frame-sample at 12 fps with max_duration=6s (k*temporal+1 rule). + 2. Bucket-resize the sampled frames TWICE: + - VAE branch (resolution=640, divisible=16, norm=0.5) + - ViT branch (resolution=616, divisible=28, norm=Qwen CLIP) + 3. ViT branch is patchified to (num_patches, 1176) for direct + consumption by Qwen2.5-VL. + + Returns: + vae_video — (C, T_lat_in, H_vae, W_vae) float tensor in + model-friendly [-1, 1] (mean=0.5, std=0.5). + vit_pixels — (num_patches, 1176) float tensor. + vit_grid_thw — (1, 3) long tensor with pre-merge grid (gt, gh, gw). + sampled_T — number of RGB frames after the temporal sampler. + """ + import numpy as _np + + if isinstance(frames_thwc, torch.Tensor): + frames_thwc = frames_thwc.detach().cpu().numpy() + if frames_thwc.dtype != _np.uint8: + # Allow [-1,1] / [0,1] float arrays; convert back to uint8 so the + # bucket transforms operate on the same domain as upstream. + arr = frames_thwc + if arr.max() <= 1.5: + arr = (arr * 255.0).clip(0, 255).astype(_np.uint8) + else: + arr = arr.astype(_np.uint8) + frames_thwc = arr + + T_in = frames_thwc.shape[0] + indices = cls._lance_sample_frame_indices(T_in, origin_fps) + sampled = frames_thwc[indices] # (T_sampled, H, W, 3) + + vae_video = cls._lance_bucket_resize_video( + sampled, + resolution=LANCE_VIDEO_VAE_RESOLUTION, + stride=LANCE_VIDEO_BUCKET_STRIDE, + divisible_crop=LANCE_VIDEO_VAE_DIVISIBLE_CROP, + mean=0.5, + std=0.5, + ) + vit_video = cls._lance_bucket_resize_video( + sampled, + resolution=LANCE_VIDEO_VIT_RESOLUTION, + stride=LANCE_VIDEO_BUCKET_STRIDE, + divisible_crop=LANCE_VIDEO_VIT_DIVISIBLE_CROP, + mean=LANCE_VIT_NORM_MEAN, + std=LANCE_VIT_NORM_STD, + ) + # Qwen2.5-VL ViT's temporal patch size is 2 → T must be even. + # Upstream pads by repeating the last frame for odd T (see + # ``validation_dataset.py:266``): + # ``video_tensor = torch.cat([video_tensor, last_frame], dim=1)`` + if vit_video.shape[1] % 2 == 1: + vit_video = torch.cat([vit_video, vit_video[:, -1:]], dim=1) + vit_pixels, vit_grid_thw = cls._lance_patchify_vit_video(vit_video) + return vae_video, vit_pixels, vit_grid_thw, len(indices) + @staticmethod def _qwen_vl_processor_call(processor, image): - """Invoke the Qwen2-VL image processor and return ``(pixel_values, grid_thw)``.""" - proc_out = processor(images=image, return_tensors="pt") + """Invoke the Qwen2-VL image processor and return ``(pixel_values, grid_thw)``. + + Applies upstream Lance's BucketResize+DivisibleCrop BEFORE calling the + standard Qwen2VLImageProcessor. After the pre-resize the input dims + are already divisible by ``patch_size * spatial_merge_size`` and within + the processor's pixel bounds, so the processor's own smart-resize is a + no-op — the ViT sees the exact bucket-cropped pixels upstream uses. + """ + prepped = LanceBagel._lance_bucket_resize_pil(image) + proc_out = processor(images=prepped, return_tensors="pt") pixel_values = proc_out["pixel_values"] grid_thw = proc_out["image_grid_thw"] return pixel_values, grid_thw @@ -277,16 +657,127 @@ class LanceBagel(Bagel): gen_input["packed_text_position_ids"] = self._mrope_broadcast(gen_input["packed_text_position_ids"]) return gen_input, newlens, new_rope - def prepare_vae_latent(self, *args, **kwargs): - gen_input = super().prepare_vae_latent(*args, **kwargs) - if "packed_position_ids" in gen_input: - gen_input["packed_position_ids"] = self._mrope_broadcast(gen_input["packed_position_ids"]) + def _per_token_mrope_for_vae_latent(self, image_sizes, curr_rope) -> torch.Tensor: + """Build per-token Qwen2.5-VL mRoPE positions for the gen latent block. + + Matches upstream Lance (``Lance.validation_gen_KVcache`` → + ``get_rope_index``): + + start_of_image -> (P-1, P-1, P-1) + latent (hi, wi) -> (P, P+hi, P+wi) + end_of_image -> (P+max(h,w), ..., ...) + + where ``P = curr_position_id`` is the position counter AFTER the text + prefix (BAGEL parent already passes ``gen_context['ropes']``). The + ``-1`` on start_of_image matches upstream's get_rope_index layout: the + start marker sits one position before the latent block (which is + anchored at P), and the end marker sits ``max(h,w)`` positions past. + + Without per-token (h, w) variation, attention can't see the spatial + layout → image PSNR collapses ~6 dB after 30 steps. + """ + pos_t: list[int] = [] + pos_h: list[int] = [] + pos_w: list[int] = [] + for (H, W), P in zip(image_sizes, curr_rope): + h = H // self.latent_downsample + w = W // self.latent_downsample + # Upstream Lance (``shift_position_ids`` + ``get_rope_index``) + # layout for the gen latent block: + # start_of_image -> (P, P, P) + # latent (hi, wi) -> (P+1, P+1+hi, P+1+wi) + # end_of_image -> (P+max+1, P+max+1, P+max+1) + # where ``P = curr_position_id`` is the rope counter AFTER the + # text+ViT prefix (i.e., :attr:`gen_context['ropes']` snapshot + # taken before the VAE-ref prefill in ``_forward_image_edit``). + # This matches :meth:`_lance_native_prepare_vae_images` so the + # gen latent occupies the SAME rope range as the VAE-ref block + # (modality==1 / modality==2 share rope per upstream's + # ``i_sample_modality==2`` branch in ``shift_position_ids``). + max_hw = max(h, w) + pos_t.append(P) + pos_h.append(P) + pos_w.append(P) + for hi in range(h): + for wi in range(w): + pos_t.append(P + 1) + pos_h.append(P + 1 + hi) + pos_w.append(P + 1 + wi) + end_p = P + max_hw + 1 + pos_t.append(end_p) + pos_h.append(end_p) + pos_w.append(end_p) + return torch.stack( + [ + torch.tensor(pos_t, dtype=torch.long), + torch.tensor(pos_h, dtype=torch.long), + torch.tensor(pos_w, dtype=torch.long), + ], + dim=0, + ) + + def _per_token_mrope_for_video_latent(self, video_shapes, curr_rope) -> torch.Tensor: + """3-D analogue of :meth:`_per_token_mrope_for_vae_latent` for video. + + Matches upstream Lance's ``get_rope_index`` for ``modality==noise`` + (and ``modality==ref_source`` since they share rope per + ``shift_position_ids``). The temporal axis is amplified by + ``LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID`` so neighbouring + latent frames are well-separated, matching the upstream Qwen2.5-VL + rope convention. Verified against upstream's rope dump for + video_edit at video_480p (call09 cond noise iter 0): + + start_of_image -> (P, P, P) + latent (t,h,w) -> (P+1+t*2, P+1+h, P+1+w) + end_of_image -> (P+max+1, P+max+1, P+max+1) + + where ``max = max(t_lat*2-1, h_lat-1, w_lat-1) + 1`` matches the + end marker placement upstream uses (call09 final token at 159 for + T_lat=12, H_lat=35, W_lat=47 → max(22, 34, 46)+1=47, start=111, + end=158... actual upstream end=159, suggesting +2 from the body's + max). We mirror :meth:`_lance_native_prepare_vae_images` exactly + so the VAE_ref prefill and the gen noise share the same rope range. + """ + downsample_t = int(getattr(self.config.vae_config, "downsample_temporal", 4)) + t_scale = LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID + pos_t: list[int] = [] + pos_h: list[int] = [] + pos_w: list[int] = [] + for (T, H, W), P in zip(video_shapes, curr_rope): + h = H // self.latent_downsample + w = W // self.latent_downsample + t = (T - 1) // downsample_t + 1 + max_thw = max(int((t - 1) * t_scale), h - 1, w - 1) + 1 + pos_t.append(P) + pos_h.append(P) + pos_w.append(P) + for ti in range(t): + for hi in range(h): + for wi in range(w): + pos_t.append(P + 1 + int(ti * t_scale)) + pos_h.append(P + 1 + hi) + pos_w.append(P + 1 + wi) + end_p = P + max_thw + 1 + pos_t.append(end_p) + pos_h.append(end_p) + pos_w.append(end_p) + return torch.stack( + [ + torch.tensor(pos_t, dtype=torch.long), + torch.tensor(pos_h, dtype=torch.long), + torch.tensor(pos_w, dtype=torch.long), + ], + dim=0, + ) + + def prepare_vae_latent(self, curr_kvlens, curr_rope, image_sizes, new_token_ids): + gen_input = super().prepare_vae_latent(curr_kvlens, curr_rope, image_sizes, new_token_ids) + gen_input["packed_position_ids"] = self._per_token_mrope_for_vae_latent(image_sizes, curr_rope) return gen_input - def prepare_vae_latent_cfg(self, *args, **kwargs): - gen_input = super().prepare_vae_latent_cfg(*args, **kwargs) - if "cfg_packed_position_ids" in gen_input: - gen_input["cfg_packed_position_ids"] = self._mrope_broadcast(gen_input["cfg_packed_position_ids"]) + def prepare_vae_latent_cfg(self, curr_kvlens, curr_rope, image_sizes): + gen_input = super().prepare_vae_latent_cfg(curr_kvlens, curr_rope, image_sizes) + gen_input["cfg_packed_position_ids"] = self._per_token_mrope_for_vae_latent(image_sizes, curr_rope) return gen_input def prepare_start_tokens(self, *args, **kwargs): @@ -301,26 +792,33 @@ class LanceBagel(Bagel): gen_input[k] = self._mrope_broadcast(gen_input[k]) return gen_input - def prepare_vit_videos(self, curr_kvlens, curr_rope, videos, new_token_ids): - """Multi-frame ViT prefill for the ``x2t_video`` understanding path. + def prepare_vit_videos( + self, + curr_kvlens, + curr_rope, + videos, + new_token_ids, + precomputed_vit=None, + ): + """Multi-frame ViT prefill for the ``x2t_video`` / ``video_edit`` paths. ``videos`` is a list of per-request video tensors / numpy arrays of - shape ``(T, H, W, 3)``. We call the Qwen2-VL **video** processor - (stored on ``self._lance_video_processor``) which produces - ``pixel_values_videos`` already in packed - ``(num_patches_flat_3d, patch_features)`` layout plus a 3-D - ``video_grid_thw`` (``[T_lat, H_patches, W_patches]``). We forward the - same pixel-values tensor to the Qwen2.5-VL ViT (wrapped by - :class:`LanceQwen2_5_VLNaViTWrapper`), set the wrapper's pending - grid, and place the post-merger tokens in the LLM sequence with - proper 3-D mRoPE positions ``(P + t, P + h, P + w)`` over the - ``T_lat × H_lat × W_lat`` post-merge grid. + shape ``(T, H, W, 3)``. By default the Qwen2-VL video processor is + used to convert each video to ``(pixel_values_videos, video_grid_thw)``. + For ``video_edit`` precision matching, the pipeline may pre-compute the + upstream-style BucketResize output and pass it via ``precomputed_vit`` + — a list of ``(pixel_values, grid_thw)`` per video, in which case the + processor call is skipped. """ processor = getattr(self, "_lance_video_processor", None) - if processor is None: + if processor is None and precomputed_vit is None: raise RuntimeError( - "LanceBagel.prepare_vit_videos requires the pipeline to set " - "``bagel._lance_video_processor`` (a Qwen2-VL-compatible video processor)." + "LanceBagel.prepare_vit_videos requires either ``precomputed_vit`` " + "or ``bagel._lance_video_processor`` (a Qwen2-VL-compatible video processor)." + ) + if precomputed_vit is not None and len(precomputed_vit) != len(videos): + raise ValueError( + f"precomputed_vit length ({len(precomputed_vit)}) must match number of videos ({len(videos)})" ) packed_vit_token_indexes: list[int] = [] @@ -341,7 +839,7 @@ class LanceBagel(Bagel): _curr = curr = 0 newlens: list[int] = [] new_rope: list[int] = [] - for video, curr_kvlen, curr_position_id in zip(videos, curr_kvlens, curr_rope): + for vi, (video, curr_kvlen, curr_position_id) in enumerate(zip(videos, curr_kvlens, curr_rope)): packed_key_value_indexes.extend(range(curr, curr + curr_kvlen)) curr += curr_kvlen @@ -352,12 +850,19 @@ class LanceBagel(Bagel): _curr += 1 per_axis_pos.append((curr_position_id, curr_position_id, curr_position_id)) - proc_out = processor(videos=video, return_tensors="pt") - # Qwen2-VL video processor: pixel_values_videos is the same packed - # (num_patches_flat, patch_features) layout as the image processor - # but with a temporal axis ``T_lat`` in video_grid_thw. - pixel_values = proc_out["pixel_values_videos"] - grid_thw = proc_out["video_grid_thw"] + if precomputed_vit is not None: + pixel_values, grid_thw = precomputed_vit[vi] + if not torch.is_tensor(grid_thw): + grid_thw = torch.as_tensor(grid_thw, dtype=torch.long) + if grid_thw.dim() == 1: + grid_thw = grid_thw.unsqueeze(0) + else: + proc_out = processor(videos=video, return_tensors="pt") + # Qwen2-VL video processor: pixel_values_videos is packed + # (num_patches_flat, patch_features) with a temporal axis + # ``T_lat`` in video_grid_thw. + pixel_values = proc_out["pixel_values_videos"] + grid_thw = proc_out["video_grid_thw"] T_lat, H, W = (int(v) for v in grid_thw[0].tolist()) num_patches_pre_merge = T_lat * H * W assert num_patches_pre_merge == pixel_values.shape[0], ( @@ -375,6 +880,14 @@ class LanceBagel(Bagel): packed_indexes.extend(range(curr, curr + num_vit_tokens)) curr += num_vit_tokens _curr += num_vit_tokens + # Upstream Lance ViT layout (verified against video_edit rope dump + # for car @ video_480p — call2 t=(1000,1046), h,w=(64,110)): + # start_of_image -> (P, P, P) + # body (t,h,w) -> (P+1+t*ts, P+1+h, P+1+w) + # end_of_image -> (P+max+2, P+max+2, P+max+2) + # where max = max(ts*(T-1), h-1, w-1) + # then the entire ViT block's t-channel is shifted to 1000+ via + # ``shift_position_ids(pos_shift=1000)`` for modality=ref_vit. h_merged = H // merge_size w_merged = W // merge_size t_scale = LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID @@ -383,9 +896,9 @@ class LanceBagel(Bagel): for wi in range(w_merged): per_axis_pos.append( ( - curr_position_id + int(ti * t_scale), - curr_position_id + hi, - curr_position_id + wi, + curr_position_id + 1 + int(ti * t_scale), + curr_position_id + 1 + hi, + curr_position_id + 1 + wi, ) ) @@ -394,7 +907,8 @@ class LanceBagel(Bagel): packed_indexes.append(curr) curr += 1 _curr += 1 - end_p = curr_position_id + max(int((T_lat - 1) * t_scale), h_merged - 1, w_merged - 1) + 1 + max_offset = max(int((T_lat - 1) * t_scale), h_merged - 1, w_merged - 1) + end_p = curr_position_id + max_offset + 2 per_axis_pos.append((end_p, end_p, end_p)) packed_seqlens.append(num_vit_tokens + 2) @@ -404,6 +918,11 @@ class LanceBagel(Bagel): pos_t = torch.tensor([p[0] for p in per_axis_pos], dtype=torch.long) pos_h = torch.tensor([p[1] for p in per_axis_pos], dtype=torch.long) pos_w = torch.tensor([p[2] for p in per_axis_pos], dtype=torch.long) + # Apply upstream's ViT t-channel shift (modality=ref_vit=4 → +1000). + # Without this, the ViT block's K/V at the t-channel collides with the + # VAE_ref / noise block in mRoPE space (same low-100s range), and + # noise-query attention picks up cross-modal leakage. + pos_t = pos_t - pos_t[0] + 1000 packed_position_ids_3d = torch.stack([pos_t, pos_h, pos_w], dim=0) generation_input = { @@ -487,21 +1006,26 @@ class LanceBagel(Bagel): packed_indexes.extend(range(curr, curr + num_img_tokens)) curr += num_img_tokens _curr += num_img_tokens - # 3-D mRoPE positions for image patches: scan in row-major over the - # post-merge grid (H/merge × W/merge). Temporal axis stays at the - # block's scalar position (single still image). + # 3-D mRoPE positions for image patches. Upstream Lance + # (``modeling/lance/lance.py::get_rope_index`` + ``shift_position_ids``) + # places ViT body tokens at ``(P+1+t, P+1+h, P+1+w)`` and the + # surrounding markers at ``(P, ...)`` (start) / ``(P+max+1, ...)`` + # (end), then SHIFTS the t-channel of the entire ViT block to + # 1000+ (modality=ref_vit=4) so ViT keys don't conflate with VAE + # ref / noise keys in mRoPE space. ``new_rope = P + max + 2``. h_merged = H // merge_size w_merged = W // merge_size + max_hw = max(h_merged, w_merged) for hi in range(h_merged): for wi in range(w_merged): - per_axis_pos.append((curr_position_id, curr_position_id + hi, curr_position_id + wi)) + per_axis_pos.append((curr_position_id + 1, curr_position_id + 1 + hi, curr_position_id + 1 + wi)) packed_text_ids.append(new_token_ids["end_of_image"]) packed_text_indexes.append(_curr) packed_indexes.append(curr) curr += 1 _curr += 1 - end_p = curr_position_id + max(h_merged - 1, w_merged - 1) + 1 + end_p = curr_position_id + max_hw + 1 per_axis_pos.append((end_p, end_p, end_p)) packed_seqlens.append(num_img_tokens + 2) @@ -511,6 +1035,13 @@ class LanceBagel(Bagel): pos_t = torch.tensor([p[0] for p in per_axis_pos], dtype=torch.long) pos_h = torch.tensor([p[1] for p in per_axis_pos], dtype=torch.long) pos_w = torch.tensor([p[2] for p in per_axis_pos], dtype=torch.long) + # Upstream Lance shifts the ViT block's t-channel into a far-away rope + # range (1000+) via ``shift_position_ids(pos_shift=1000)`` on + # modality=ref_vit (4). This makes ViT keys/queries mRoPE-orthogonal + # to noise/VAE-ref tokens along the t-channel, preventing leakage in + # the time-frequency dimension. We apply the same shift here so the + # ViT segment's K, V cache content matches upstream. + pos_t = pos_t - pos_t[0] + 1000 packed_position_ids_3d = torch.stack([pos_t, pos_h, pos_w], dim=0) generation_input = { @@ -581,6 +1112,11 @@ class LanceBagel(Bagel): packed_indexes.append(curr) curr += 1 query_curr += 1 + # Upstream Lance places gen-latent start_of_image at ``(P, P, P)`` + # (NOT ``P-1`` and NOT shifted to ViT t-range), body at + # ``(P+1+ti*t_scale, P+1+hi, P+1+wi)``, end at ``(P+max+1, ...)``. + # Verified against upstream's video_edit rope dump for T_lat=12, + # H_lat=35, W_lat=47 (call09 / cond noise iter 0). per_axis_pos.append((curr_position_id, curr_position_id, curr_position_id)) h = H // self.latent_downsample @@ -612,9 +1148,9 @@ class LanceBagel(Bagel): for ti, hi, wi in zip(tt_flat, hh_flat, ww_flat): per_axis_pos.append( ( - curr_position_id + int(ti * t_scale), - curr_position_id + hi, - curr_position_id + wi, + curr_position_id + 1 + int(ti * t_scale), + curr_position_id + 1 + hi, + curr_position_id + 1 + wi, ) ) @@ -623,9 +1159,13 @@ class LanceBagel(Bagel): packed_indexes.append(curr) curr += 1 query_curr += 1 - # end_of_image sits "past" the latent block; bump along all axes by - # max of the latent block size so subsequent text resumes cleanly. - end_p = curr_position_id + max(t - 1, h - 1, w - 1) + 1 + # end_of_image sits "past" the latent block at ``P+max+1`` on every + # axis, where ``max = max(t_scale*(t-1), h-1, w-1) + 1``. Verified + # against upstream's call09 final token at (159, 159, 159) for + # T_lat=12 (t_scale=2 → max_t=22), H_lat=35 (max_h=34), W_lat=47 + # (max_w=46), with P=111: end_p = 111 + 46 + 1 + 1 = 159. ✓ + max_thw = max(int((t - 1) * t_scale), h - 1, w - 1) + 1 + end_p = curr_position_id + max_thw + 1 per_axis_pos.append((end_p, end_p, end_p)) packed_seqlens.append(num_video_tokens + 2) @@ -654,13 +1194,19 @@ class LanceBagel(Bagel): """3-D analogue of :meth:`prepare_vae_latent_cfg` (CFG side). Mirrors :meth:`prepare_video_latent`'s mRoPE 3-D position layout - (text frame ⇒ scalar; video latent block ⇒ per-token ``(P+t, P+h, P+w)``). + EXACTLY, including the ``LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID`` + temporal scaling. Without that scaling, the cfg_text branch attends + with different rope coordinates than the cond branch (and than + upstream's ``get_rope_index``), which makes ``cfg_text_v_t`` diverge + from upstream and the CFG combination amplifies the error every + denoise step. """ downsample_t = int(getattr(self.config.vae_config, "downsample_temporal", 4)) packed_indexes, packed_key_value_indexes = [], [] per_axis_pos: list[tuple[int, int, int]] = [] query_curr = curr = 0 + t_scale = LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID for (T, H, W), curr_kvlen, curr_position_id in zip(video_shapes, curr_kvlens, curr_rope): packed_key_value_indexes.extend(range(curr, curr + curr_kvlen)) curr += curr_kvlen @@ -668,6 +1214,8 @@ class LanceBagel(Bagel): packed_indexes.append(curr) curr += 1 query_curr += 1 + # Match :meth:`prepare_video_latent` convention exactly (see + # comments there for the upstream rope layout this replicates). per_axis_pos.append((curr_position_id, curr_position_id, curr_position_id)) h = H // self.latent_downsample @@ -679,12 +1227,19 @@ class LanceBagel(Bagel): query_curr += num_video_tokens tt, hh, ww = torch.meshgrid(torch.arange(t), torch.arange(h), torch.arange(w), indexing="ij") for ti, hi, wi in zip(tt.flatten().tolist(), hh.flatten().tolist(), ww.flatten().tolist()): - per_axis_pos.append((curr_position_id + ti, curr_position_id + hi, curr_position_id + wi)) + per_axis_pos.append( + ( + curr_position_id + 1 + int(ti * t_scale), + curr_position_id + 1 + hi, + curr_position_id + 1 + wi, + ) + ) packed_indexes.append(curr) curr += 1 query_curr += 1 - end_p = curr_position_id + max(t - 1, h - 1, w - 1) + 1 + max_thw = max(int((t - 1) * t_scale), h - 1, w - 1) + 1 + end_p = curr_position_id + max_thw + 1 per_axis_pos.append((end_p, end_p, end_p)) pos_t = torch.tensor([p[0] for p in per_axis_pos], dtype=torch.long) @@ -776,15 +1331,32 @@ class LanceBagel(Bagel): packed_indexes.extend(range(curr, curr + num_image_tokens)) curr += num_image_tokens query_curr += num_image_tokens + # Upstream Lance places VAE-ref body at ``(P+1+t*t_scale, P+1+h, P+1+w)`` + # with the start marker at ``(P, P, P)`` and the end marker at + # ``(P+max+1, ...)``; gen-latent tokens (noise) share the SAME + # rope range via the modality_map==1/2 trick, so this is the + # convention used by :meth:`_per_token_mrope_for_vae_latent` (2D) + # and :meth:`_per_token_mrope_for_video_latent` (3D) below. + # For video, ``t_scale = LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID`` + # matches upstream Qwen2.5-VL's temporal-axis scaling; for still + # images we leave the t-channel scalar (T=1 → ``ti=0`` anyway). + t_scale = LANCE_TOKENS_PER_SECOND * LANCE_SECONDS_PER_GRID if is_video else 1 + max_thw = max(int((t_lat - 1) * t_scale), h_lat - 1, w_lat - 1) + 1 for ti, hi, wi in zip(tt.flatten().tolist(), hh.flatten().tolist(), ww.flatten().tolist()): - per_axis_pos.append((curr_position_id + ti, curr_position_id + hi, curr_position_id + wi)) + per_axis_pos.append( + ( + curr_position_id + 1 + int(ti * t_scale), + curr_position_id + 1 + hi, + curr_position_id + 1 + wi, + ) + ) packed_text_ids.append(new_token_ids["end_of_image"]) packed_text_indexes.append(query_curr) packed_indexes.append(curr) curr += 1 query_curr += 1 - end_p = curr_position_id + max(t_lat - 1, h_lat - 1, w_lat - 1) + 1 + end_p = curr_position_id + max_thw + 1 per_axis_pos.append((end_p, end_p, end_p)) packed_seqlens.append(num_image_tokens + 2) @@ -824,15 +1396,25 @@ class LanceBagel(Bagel): return generation_input, newlens, new_rope def prepare_vae_images(self, curr_kvlens, curr_rope, images, transforms, new_token_ids, timestep=0): - """No-op VAE prefill for the x2t / x2t_video path. + """VAE prefill router. - Lance's understanding paths route image / video context through the - Qwen2.5-VL ViT only. BAGEL's pipeline runs ``prepare_vae_images`` - unconditionally for any image input, so this default override - short-circuits it. :meth:`_forward_image_edit` / - :meth:`_forward_video_edit` call :meth:`_lance_native_prepare_vae_images` - directly when they actually need Lance-style VAE prefill. + - When ``images`` is non-empty (image_edit / video_edit path on the + image side): delegate to :meth:`_lance_native_prepare_vae_images` + which emits Lance's 3-D mRoPE positions and a real VAE prefill, + letting BAGEL's parent ``image_edit`` flow handle the rest. + - When ``images`` is empty (t2i / x2t paths): short-circuit with the + no-op output, mirroring BAGEL's "no image to prefill" sentinel. """ + if images: + return self._lance_native_prepare_vae_images( + curr_kvlens=curr_kvlens, + curr_rope=curr_rope, + images=images, + transforms=transforms, + new_token_ids=new_token_ids, + timestep=timestep, + is_video=False, + ) generation_input = { "padded_images": torch.empty(0, 3, 0, 0), "patchified_vae_latent_shapes": [], @@ -865,6 +1447,7 @@ class LanceBagel(Bagel): packed_indexes=None, key_values_lens=None, packed_key_value_indexes=None, + precomputed_latent=None, ): """Lance-native VAE prefill that *actually* scatters the encoded latents into the LLM query sequence. @@ -888,12 +1471,28 @@ class LanceBagel(Bagel): if ( packed_text_ids is None or packed_text_ids.numel() == 0 - or padded_images is None - or padded_images.numel() == 0 + or (precomputed_latent is None and (padded_images is None or padded_images.numel() == 0)) ): return past_key_values - padded_latent = vae_model.encode(padded_images) + # ``vae_model.encode`` samples from the posterior via + # ``mu + std * randn_like(std)`` (Wan2.2 VAE's ``reparameterize`` path + # — both upstream and vllm-omni default to ``use_sample=True``). Each + # call therefore returns a DIFFERENT latent. Upstream's image_edit + # encodes the reference image ONCE and reuses the resulting latent + # for both the cond and cfg_text branches; calling encode separately + # per branch made vllm-omni's gen vs cfg VAE-ref K cache diverge by + # ~6% rel_l2, which cascaded into ~33% v_t divergence at iter 0 on + # the cond branch (cfg matched within bf16 noise because it sees the + # SAME ref VAE K in both implementations — only gen-vs-cfg drift + # matters for the qualitative edit, since the noise query attends to + # both branches' caches at every iter). The pipeline now encodes + # the ref once and threads the encoded latent into both prefill + # calls via ``precomputed_latent``. + if precomputed_latent is not None: + padded_latent = precomputed_latent + else: + padded_latent = vae_model.encode(padded_images) p = self.latent_patch_size packed_latent_list = [] for latent, shape in zip(padded_latent, patchified_vae_latent_shapes): @@ -921,6 +1520,13 @@ class LanceBagel(Bagel): packed_query_sequence[packed_text_indexes] = packed_text_embed.to(packed_query_sequence.dtype) packed_query_sequence[packed_vae_token_indexes] = packed_latent.to(packed_query_sequence.dtype) + # Upstream Lance routes the VAE-ref block through MoE_GEN because + # ``packed_gen_token_indexes`` in ``Lance.validation_gen_KVcache`` + # is set to ``current_vae_token_indexes_local`` — i.e., it includes + # BOTH ref VAE tokens (modality=2 ref_source) AND noise tokens + # (modality=1). For the VAE-ref segment forward, the mask + # ``packed_gen_token_indexes ∈ [current_cond_start, current_cond_end)`` + # filters in the VAE-ref body tokens → ``mode_ = "gen"``. extra_inputs = {} if self.use_moe: extra_inputs = { diff --git a/vllm_omni/diffusion/models/lance/pipeline_lance.py b/vllm_omni/diffusion/models/lance/pipeline_lance.py index 83998e2f0..24a3631fa 100644 --- a/vllm_omni/diffusion/models/lance/pipeline_lance.py +++ b/vllm_omni/diffusion/models/lance/pipeline_lance.py @@ -77,6 +77,41 @@ from .wan_vae import LanceWanVAE logger = init_logger(__name__) +def _extract_user_instruction(rendered: str) -> str: + """Pull just the user instruction out of a chat-template-rendered prompt. + + ``render_lance_prompt`` wraps the instruction in: + + <|im_start|>system\\n{sys}<|im_end|>\\n + <|im_start|>user\\n{vision_block}{user_text}<|im_end|>\\n + <|im_start|>assistant\\n + + The ``image_edit`` pipeline rebuilds the chat template segment-by-segment + (so the rope positions and segment boundaries match upstream Lance), and + therefore needs only ``user_text`` — not the wrapped string. This helper + extracts it; if the input doesn't look templated it's returned as-is. + """ + if "<|im_start|>user" not in rendered: + return rendered.strip() + # User block sits between the user header and the next ``<|im_end|>``. + after_user = rendered.split("<|im_start|>user", 1)[1] + after_user = after_user.lstrip("\n") + user_block = after_user.split("<|im_end|>", 1)[0] + # Strip any vision_start..vision_end blocks. + out = [] + i = 0 + while i < len(user_block): + if user_block.startswith("<|vision_start|>", i): + j = user_block.find("<|vision_end|>", i) + if j == -1: + break + i = j + len("<|vision_end|>") + continue + out.append(user_block[i]) + i += 1 + return "".join(out).strip() + + @dataclass(frozen=True) class LanceDefaults: """Lance constants that upstream keeps in ``config/config_factory.py`` / @@ -342,7 +377,18 @@ class LancePipeline(BagelPipeline): fall_back_to_pt=False, ), ] - if und_enabled and not is_video and os.path.isdir(os.path.join(repo_root, _VIT_DIR)): + # Always overlay ``Qwen2.5-VL-ViT/vit.safetensors`` when the bundle + # contains it. Upstream Lance's ``inference_lance.py`` loads ViT + # weights from ``vit.safetensors`` unconditionally (line 455-458), + # using the released understanding-ViT checkpoint regardless of t2i + # vs t2v / image_edit vs video_edit. Lance_3B_Video bundles its own + # ``vit_model.*`` inside ``model.safetensors``, but loading those + # produced byte-different ViT outputs from upstream (rel_l2 ≈ 9 + # at the merger output) — the bundled video-checkpoint ViT diverges + # from the standalone vit.safetensors. Mirroring upstream's load + # order (vit.safetensors LAST so it wins) restores byte-identical + # ViT outputs for both image_edit and video_edit. + if und_enabled and os.path.isdir(os.path.join(repo_root, _VIT_DIR)): self.weights_sources.append( DiffusersPipelineLoader.ComponentSource( model_or_path=weights_model, @@ -468,6 +514,13 @@ class LancePipeline(BagelPipeline): return self._forward_x2t_video(req) if "image" in modalities and (mm_data.get("img2img") is not None or mm_data.get("image") is not None): return self._forward_image_edit(req) + # t2i falls through to BAGEL parent. Inject Lance defaults if the + # caller hasn't set them: upstream uses ``cfg_text_scale=4.0`` and + # ``cfg_vit_scale=1.0`` (= cfg_img_scale=1.0 in BAGEL terms). + sp = req.sampling_params + if not hasattr(sp, "extra_args") or sp.extra_args is None: + sp.extra_args = {} + sp.extra_args.setdefault("cfg_img_scale", 1.0) return super().forward(req) @torch.inference_mode() @@ -511,6 +564,13 @@ class LancePipeline(BagelPipeline): logger.info("Lance t2v: video_shape=%s", video_shape) cfg_text_scale = float(extra_args.get("cfg_text_scale", 4.0)) + # Upstream Lance defaults cfg_interval=[0.4, 1.0] (turn off CFG below + # t=0.4); without this vllm-omni keeps CFG on for late iters and the + # last few denoise steps diverge from upstream. cfg_renorm_type / + # cfg_renorm_min default to 'global' / 0 on both sides. + cfg_interval = tuple(extra_args.get("cfg_interval", (0.4, 1.0))) + cfg_renorm_type = str(extra_args.get("cfg_renorm_type", "global")) + cfg_renorm_min = float(extra_args.get("cfg_renorm_min", 0.0)) timestep_shift = float(extra_args.get("timestep_shift", LANCE_DEFAULTS.timestep_shift)) num_timesteps = int(req.sampling_params.num_inference_steps or LANCE_DEFAULTS.num_timesteps) @@ -613,6 +673,9 @@ class LancePipeline(BagelPipeline): timestep_shift=timestep_shift, cfg_text_scale=cfg_text_scale, cfg_img_scale=1.0, + cfg_interval=cfg_interval, + cfg_renorm_type=cfg_renorm_type, + cfg_renorm_min=cfg_renorm_min, **gen_input_lat, cfg_text_packed_position_ids=cfg_text_lat["cfg_packed_position_ids"], cfg_text_packed_query_indexes=cfg_text_lat["cfg_packed_query_indexes"], @@ -638,11 +701,30 @@ class LancePipeline(BagelPipeline): def _forward_image_edit(self, req): """Image edit (img2img): reference image + text prompt → modified image. - Mirrors :meth:`_forward_t2v` for the text/VAE prefill structure but - also runs ``_lance_native_prepare_vae_images`` for the reference - image (so the LLM sees both the Wan2.2-encoded latents and the - Qwen2.5-VL ViT context) and then a Lance image-gen loop emitting - 2-D latents for the new image. + Matches upstream Lance's image_edit prefill structure (see + ``Lance.validation_gen_KVcache`` and ``validation_dataset.py``). The + sequence is laid out as 5 prefill segments + 1 noise QUERY: + + seg1 (causal, modality=system_prompt): + ``<|im_start|>system\\n{sys}<|im_end|>\\n<|im_start|>user\\n`` + seg2 (full, modality=ref_vit): + ViT(ref) — ref image through Qwen2.5-VL ViT + seg3 (full_noise, modality=ref_source): + VAE(ref) — ref image through Wan2.2 VAE, time_embed(t=0) + seg4 (causal, modality=text): the user instruction + seg5 (causal, modality=system_prompt): + ``<|im_end|>\\n<|im_start|>assistant\\n`` + seg6 (noise QUERY): the gen latent to denoise + + cfg_text_context skips ONLY seg4 (the user text instruction) — every + other segment, including the system header, ViT and VAE prefill, the + separator and the noise query, is shared between gen_context and + cfg_text_context. The rope counter still advances by seg4's length + for cfg_text_context so that seg5's rope position matches between + the two branches. + + cfg_img is OFF by default (``cfg_img_scale=1.0`` matches upstream's + ``cfg_vit_scale=1.0``). """ import numpy as np @@ -650,10 +732,18 @@ class LancePipeline(BagelPipeline): from vllm_omni.diffusion.data import DiffusionOutput from .lance_transformer import NaiveCache + from .prompts import SYSTEM_PROMPTS first_prompt = req.prompts[0] assert isinstance(first_prompt, dict), "image_edit requires dict-style prompt" - prompt = first_prompt.get("prompt") or "" + # The caller may pass either a raw user instruction (e.g. "Remove the + # hat from the painting.") OR an already-rendered template string + # (with system + chat template). Extract just the user instruction + # so we can rebuild the segments cleanly. + user_text = first_prompt.get("user_text") + rendered = first_prompt.get("prompt") or "" + if not user_text: + user_text = _extract_user_instruction(rendered) mm_data = first_prompt.get("multi_modal_data") or {} image_input = mm_data.get("img2img") or mm_data.get("image") if image_input is None: @@ -666,7 +756,10 @@ class LancePipeline(BagelPipeline): sp_extra = getattr(req.sampling_params, "extra_args", {}) or {} extra_args = {**extra_args, **sp_extra} cfg_text_scale = float(extra_args.get("cfg_text_scale", 4.0)) - cfg_img_scale = float(extra_args.get("cfg_img_scale", 1.5)) + cfg_img_scale = float(extra_args.get("cfg_img_scale", 1.0)) + cfg_interval = tuple(extra_args.get("cfg_interval", (0.4, 1.0))) + cfg_renorm_type = str(extra_args.get("cfg_renorm_type", "global")) + cfg_renorm_min = float(extra_args.get("cfg_renorm_min", 0.0)) timestep_shift = float(extra_args.get("timestep_shift", LANCE_DEFAULTS.timestep_shift)) num_timesteps = int(req.sampling_params.num_inference_steps or LANCE_DEFAULTS.num_timesteps) @@ -692,7 +785,7 @@ class LancePipeline(BagelPipeline): image_input = [_resize_to_stride(im) for im in image_input] resized_w, resized_h = image_input[0].size image_shape = (resized_h, resized_w) - logger.info("Lance image_edit: ref image %dx%d", resized_w, resized_h) + logger.info("Lance image_edit: ref image %dx%d, user_text=%r", resized_w, resized_h, user_text) def vae_transforms(img): arr = torch.from_numpy(np.array(img)).float() / 127.5 - 1.0 @@ -706,123 +799,160 @@ class LancePipeline(BagelPipeline): if self.device.type == "cuda": torch.cuda.manual_seed(req.sampling_params.seed) - # Match upstream Lance's edit position-id convention: input reference - # VAE block and output gen latent share rope range ``[0, max_axis]``, - # while text / ViT prefill are shifted by ``_EDIT_SHIFT`` so they - # never collide. Without this, the model sees the reference at - # different rope coordinates than the gen target and effectively - # regenerates instead of editing. - _EDIT_SHIFT = 10000 gen_context = { "kv_lens": [0], "ropes": [0], "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), } - - # VAE prefill (Lance-native, 3-D mRoPE positions) — rope starts at 0 - # to align with the gen latent below. - vae_input, newlens_vae, _ = self.bagel._lance_native_prepare_vae_images( - curr_kvlens=gen_context["kv_lens"], - curr_rope=gen_context["ropes"], - images=image_input, - transforms=vae_transforms, - new_token_ids=self.new_token_ids, - is_video=False, - ) - for k, v in vae_input.items(): - if torch.is_tensor(v): - vae_input[k] = v.to(self.device) - with torch.autocast( - device_type=self.device.type, - enabled=self.device.type != "cpu", - dtype=self.od_config.dtype, - ): - gen_context["past_key_values"] = self.bagel.forward_cache_update_vae( - self.vae, gen_context["past_key_values"], **vae_input - ) - gen_context["kv_lens"] = newlens_vae - # Shift rope out of the gen-latent range for the following text / ViT - # prefills; we snap back to 0 when constructing the gen latent. - gen_context["ropes"] = [_EDIT_SHIFT] - - # ViT prefill (Lance Qwen2.5-VL with packed flat patches). - vit_input, newlens_vit, new_rope_vit = self.bagel.prepare_vit_images( - curr_kvlens=gen_context["kv_lens"], - curr_rope=gen_context["ropes"], - images=image_input, - transforms=vit_transforms, - new_token_ids=self.new_token_ids, - ) - for k, v in vit_input.items(): - if torch.is_tensor(v): - vit_input[k] = v.to(self.device) - with torch.autocast( - device_type=self.device.type, - enabled=self.device.type != "cpu", - dtype=self.od_config.dtype, - ): - gen_context["past_key_values"] = self.bagel.forward_cache_update_vit( - gen_context["past_key_values"], **vit_input - ) - gen_context["kv_lens"] = newlens_vit - gen_context["ropes"] = new_rope_vit - - # Text prefill. - text_input, newlens_text, new_rope_text = self.bagel.prepare_prompts( - curr_kvlens=gen_context["kv_lens"], - curr_rope=gen_context["ropes"], - prompts=[prompt], - tokenizer=self.tokenizer, - new_token_ids=self.new_token_ids, - ) - for k, v in text_input.items(): - if torch.is_tensor(v): - text_input[k] = v.to(self.device) - with torch.autocast( - device_type=self.device.type, - enabled=self.device.type != "cpu", - dtype=self.od_config.dtype, - ): - gen_context["past_key_values"] = self.bagel.forward_cache_update_text( - gen_context["past_key_values"], **text_input - ) - gen_context["kv_lens"] = newlens_text - gen_context["ropes"] = new_rope_text - - # CFG text branch (unconditional text). cfg_text_context = { "kv_lens": [0], "ropes": [0], "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), } - if cfg_text_scale > 1.0: - neg_input, neg_newlens, neg_rope = self.bagel.prepare_prompts( - curr_kvlens=cfg_text_context["kv_lens"], - curr_rope=cfg_text_context["ropes"], - prompts=[str(extra_args.get("negative_prompt") or "")], - tokenizer=self.tokenizer, + + autocast_kwargs = dict( + device_type=self.device.type, + enabled=self.device.type != "cpu", + dtype=self.od_config.dtype, + ) + + # ----- prefill helpers (no bos/eos wrapping — that's only for free- + # form prompts; the segmented template provides its own markers) ----- + def _raw_text_prefill(ctx, text_str): + """Prefill `text_str` as-is — no automatic bos/eos. + + ``Bagel.prepare_prompts`` wraps with ``[bos] + ids + [eos]`` which + is fine for a single user prompt but breaks segment-by-segment + templating where each segment already carries its own framing + tokens. This helper tokenizes raw and feeds the LLM directly. + """ + text_ids = self.tokenizer.encode(text_str, add_special_tokens=False) + if not text_ids: + return + curr_kvlen = ctx["kv_lens"][0] + curr_rope = ctx["ropes"][0] + seq_len = len(text_ids) + inp = { + "text_token_lens": torch.tensor([seq_len], dtype=torch.int, device=self.device), + "packed_text_ids": torch.tensor(text_ids, dtype=torch.long, device=self.device), + "packed_text_position_ids": torch.arange( + curr_rope, curr_rope + seq_len, dtype=torch.long, device=self.device + ), + "packed_text_indexes": torch.arange( + curr_kvlen, curr_kvlen + seq_len, dtype=torch.long, device=self.device + ), + "packed_key_value_indexes": torch.arange(0, curr_kvlen, dtype=torch.long, device=self.device), + "key_values_lens": torch.tensor([curr_kvlen], dtype=torch.int, device=self.device), + } + with torch.autocast(**autocast_kwargs): + ctx["past_key_values"] = self.bagel.forward_cache_update_text(ctx["past_key_values"], **inp) + ctx["kv_lens"] = [curr_kvlen + seq_len] + ctx["ropes"] = [curr_rope + seq_len] + + def _vit_prefill(ctx): + inp, new_kvlens, new_rope = self.bagel.prepare_vit_images( + curr_kvlens=ctx["kv_lens"], + curr_rope=ctx["ropes"], + images=image_input, + transforms=vit_transforms, new_token_ids=self.new_token_ids, ) - for k, v in neg_input.items(): + for k, v in inp.items(): if torch.is_tensor(v): - neg_input[k] = v.to(self.device) - with torch.autocast( - device_type=self.device.type, - enabled=self.device.type != "cpu", - dtype=self.od_config.dtype, - ): - cfg_text_context["past_key_values"] = self.bagel.forward_cache_update_text( - cfg_text_context["past_key_values"], **neg_input - ) - cfg_text_context["kv_lens"] = neg_newlens - cfg_text_context["ropes"] = neg_rope + inp[k] = v.to(self.device) + with torch.autocast(**autocast_kwargs): + ctx["past_key_values"] = self.bagel.forward_cache_update_vit(ctx["past_key_values"], **inp) + ctx["kv_lens"] = new_kvlens + ctx["ropes"] = new_rope - # ---- 2-D latent init (image output) ---- - # Gen latent uses rope=[0] so its tokens share position coordinates - # with the reference image's VAE block above (edit alignment). + # Encode the reference image ONCE so both gen and cfg branches share + # the same posterior sample. ``LanceWanVAE.encode`` uses + # ``mu + std * randn_like(std)`` (Wan2.2 VAE's reparameterize path) + # which yields a different latent every call; calling it separately + # per branch drifts gen-branch K, V cache from cfg-branch by ~6% + # rel_l2 even though both should see the same ref. + _ref_padded_latent_cache = {"v": None} + + def _vae_prefill(ctx): + inp, new_kvlens, new_rope = self.bagel._lance_native_prepare_vae_images( + curr_kvlens=ctx["kv_lens"], + curr_rope=ctx["ropes"], + images=image_input, + transforms=vae_transforms, + new_token_ids=self.new_token_ids, + is_video=False, + ) + for k, v in inp.items(): + if torch.is_tensor(v): + inp[k] = v.to(self.device) + # Compute the ref VAE latent on the first prefill call; reuse + # for subsequent branches. + if _ref_padded_latent_cache["v"] is None: + _ref_padded_latent_cache["v"] = self.vae.encode(inp["padded_images"].to(self.device)) + inp["precomputed_latent"] = _ref_padded_latent_cache["v"] + with torch.autocast(**autocast_kwargs): + ctx["past_key_values"] = self.bagel.forward_cache_update_vae(self.vae, ctx["past_key_values"], **inp) + ctx["kv_lens"] = new_kvlens + ctx["ropes"] = new_rope + + # ----- prefill segments in upstream order ----- + sys_prompt = SYSTEM_PROMPTS[("image_edit", "image")] + seg1_str = f"<|im_start|>system\n{sys_prompt}<|im_end|>\n<|im_start|>user\n" + seg5_str = "<|im_end|>\n<|im_start|>assistant\n" + + # seg1: system + user header (shared) + _raw_text_prefill(gen_context, seg1_str) + if cfg_text_scale > 1.0: + _raw_text_prefill(cfg_text_context, seg1_str) + + # seg2: ViT(ref) (shared) + _vit_prefill(gen_context) + if cfg_text_scale > 1.0: + _vit_prefill(cfg_text_context) + + # seg3: VAE(ref) with timestep=0 (shared) + # + # IMPORTANT: upstream Lance places the gen latent (the noise QUERY) + # at the SAME mRoPE positions as the ref VAE — they share the same + # spatial grid (T_lat × H_lat × W_lat), and the model treats them + # as two timesteps of the same latent location. Concretely, upstream + # ``get_rope_index`` emits ``[base_t, base_h+hi, base_w+wi]`` for + # both the VAE cond block AND the noise block, anchored at the SAME + # base. This is the inductive bias that lets the model "edit" the + # ref image — without this overlap the model has no way to map noise + # tokens to ref tokens. + # + # We therefore snapshot ``ropes`` BEFORE the VAE prefill and reuse + # that as the gen latent's ``curr_rope`` below. + rope_before_vae = gen_context["ropes"][0] + cfg_rope_before_vae = cfg_text_context["ropes"][0] if cfg_text_scale > 1.0 else rope_before_vae + _vae_prefill(gen_context) + if cfg_text_scale > 1.0: + _vae_prefill(cfg_text_context) + + # seg4: user instruction (gen only). Upstream Lance's cfg_text + # branch is a SHORTER sequence with the user_text segment REMOVED + # (see ``uncond_split_pro_kvcache`` + ``get_rope_index`` on the + # filtered text_ids), so cfg_text's rope counter MUST NOT advance + # past the user_text region. seg5 (separator) therefore lands at + # different absolute rope positions in gen vs cfg branches — that's + # the intended behavior. + _raw_text_prefill(gen_context, user_text) + + # seg5: separator + assistant header (gen + cfg). Each branch + # places it at its OWN current rope. + _raw_text_prefill(gen_context, seg5_str) + if cfg_text_scale > 1.0: + _raw_text_prefill(cfg_text_context, seg5_str) + + # -- (4) Gen latent at the SAME rope position as the ref VAE block. + # See the comment on `rope_before_vae` above. Note the KV cache + # still contains all prefilled segments (system, ViT, ref VAE, user + # text, separator); only the gen latent's QUERY rope coordinates are + # rewound to overlap with the ref VAE. gen_input_lat = self.bagel.prepare_vae_latent( curr_kvlens=gen_context["kv_lens"], - curr_rope=[0], + curr_rope=[rope_before_vae], image_sizes=[image_shape], new_token_ids=self.new_token_ids, ) @@ -831,34 +961,33 @@ class LancePipeline(BagelPipeline): gen_input_lat[k] = v.to(self.device) cfg_text_lat = self.bagel.prepare_vae_latent_cfg( curr_kvlens=cfg_text_context["kv_lens"], - curr_rope=[0], - image_sizes=[image_shape], - ) - cfg_img_lat = self.bagel.prepare_vae_latent_cfg( - curr_kvlens=gen_context["kv_lens"], - curr_rope=[0], + curr_rope=[cfg_rope_before_vae], image_sizes=[image_shape], ) for k, v in cfg_text_lat.items(): if torch.is_tensor(v): cfg_text_lat[k] = v.to(self.device) - for k, v in cfg_img_lat.items(): - if torch.is_tensor(v): - cfg_img_lat[k] = v.to(self.device) - with torch.autocast( - device_type=self.device.type, - enabled=self.device.type != "cpu", - dtype=self.od_config.dtype, - ): + # cfg_img branch: keep off (cfg_img_scale=1.0 matches upstream + # ``cfg_vit_scale=1.0``). generate_image needs the cfg_img_* args + # but skips the branch when scale<=1.0; just point them at the + # cond's lat metadata so shapes/types are fine. + cfg_img_lat = cfg_text_lat + + with torch.autocast(**autocast_kwargs): latents, *_ = self.bagel.generate_image( past_key_values=gen_context["past_key_values"], - cfg_text_past_key_values=cfg_text_context["past_key_values"], + cfg_text_past_key_values=cfg_text_context["past_key_values"] + if cfg_text_scale > 1.0 + else gen_context["past_key_values"], cfg_img_past_key_values=gen_context["past_key_values"], num_timesteps=num_timesteps, timestep_shift=timestep_shift, cfg_text_scale=cfg_text_scale, cfg_img_scale=cfg_img_scale, + cfg_interval=cfg_interval, + cfg_renorm_type=cfg_renorm_type, + cfg_renorm_min=cfg_renorm_min, **gen_input_lat, cfg_text_packed_position_ids=cfg_text_lat["cfg_packed_position_ids"], cfg_text_packed_query_indexes=cfg_text_lat["cfg_packed_query_indexes"], @@ -881,179 +1010,275 @@ class LancePipeline(BagelPipeline): def _forward_video_edit(self, req): """Video edit: reference video + text prompt → modified video. - Lance-native VAE prefill (multi-frame Wan2.2 encoding) + Qwen2.5-VL - ViT prefill + 3-D latent denoising + ``LanceWanVAE.decode_video``. - Mirrors :meth:`_forward_image_edit` but with the temporal axis - threaded through both prefill and gen latent. + Mirrors :meth:`_forward_image_edit`'s segmented prefill layout with + a temporal axis threaded through both prefill and the gen latent. + Verified against upstream Lance's ``Lance.validation_gen_KVcache`` for + ``--task video_edit --resolution video_480p``: + + seg1 (causal, modality=system_prompt): + ``<|im_start|>system\\n{sys}<|im_end|>\\n<|im_start|>user\\n`` + seg2 (full, modality=ref_vit): + ViT(video) — ref video through Qwen2.5-VL ViT, t-axis +1000 + seg3 (full_noise, modality=ref_source): + VAE(video) — ref video through Wan2.2 VAE, time_embed(t=0). + Rope shares range with the gen noise latent (modality 1==2 + trick) so the model can align ref↔output per ``(t,h,w)``. + seg4 (causal, modality=text): the user instruction + seg5 (causal, modality=system_prompt): + ``<|im_end|>\\n<|im_start|>assistant\\n`` + seg6 (noise QUERY): 3-D gen latent at the SAME rope as seg3. + + cfg_text_context skips ONLY seg4 (user text), matching upstream's + ``uncond_split_pro_kvcache``. cfg_img defaults to off + (``cfg_img_scale=1.0`` → upstream's ``cfg_vit_scale=1.0``). + + Replaces the previous shifted-rope layout (``_EDIT_SHIFT=10000``) + which kept ViT/text at a far-away rope range — that caused 15.5% + rel_l2 divergence at layer 0 output (cond) vs upstream because the + noise query attended to a cached K/V at rope (0..47) while upstream + attends at rope (111..159). """ + import numpy as _np + from vllm_omni.diffusion.data import DiffusionOutput from .lance_transformer import NaiveCache + from .prompts import SYSTEM_PROMPTS first_prompt = req.prompts[0] assert isinstance(first_prompt, dict), "video_edit requires dict-style prompt" - prompt = first_prompt.get("prompt") or "" + user_text = first_prompt.get("user_text") + rendered = first_prompt.get("prompt") or "" + if not user_text: + user_text = _extract_user_instruction(rendered) mm_data = first_prompt.get("multi_modal_data") or {} video_input = mm_data.get("video") if video_input is None: raise ValueError("video_edit requires multi_modal_data.video.") - if isinstance(video_input, str): - import imageio.v3 as iio - - video_input = iio.imread(video_input) - # ``video_input`` now: ndarray (T, H, W, 3). Permute to (C, T, H, W) for VAE. - import numpy as _np - - if isinstance(video_input, _np.ndarray): - video_chw = torch.from_numpy(video_input).float() / 127.5 - 1.0 # (T, H, W, 3) - video_chw = video_chw.permute(3, 0, 1, 2).contiguous() # (C, T, H, W) - elif torch.is_tensor(video_input): - video_chw = video_input.float() - else: - raise ValueError(f"Unsupported video_input type {type(video_input)}") - T = int(video_chw.shape[1]) - H = int(video_chw.shape[2]) - W = int(video_chw.shape[3]) - + # Resolve to raw (T, H, W, 3) uint8 ndarray + origin_fps so the + # upstream-style bucket resize + frame sampler can replicate the + # exact preprocessing path. If the caller passed an already-decoded + # ndarray with no FPS, default to 24fps (matches the official + # video_edit examples). extra_args = first_prompt.get("extra_args") or {} sp_extra = getattr(req.sampling_params, "extra_args", {}) or {} extra_args = {**extra_args, **sp_extra} + origin_fps_default = float(extra_args.get("origin_fps", 24.0)) + # Path-only fast path: use decord (matches upstream Lance's + # ``data/datasets_custom/validation_dataset.py`` which decodes via + # ``decord.VideoReader``). decord vs cv2 produce different YUV→RGB + # conversions (~0.1 RGB units of max diff), and that propagates into + # both the VAE and ViT pixel inputs. Falling back to cv2 only when + # decord is unavailable keeps single-machine parity with upstream + # while not requiring decord in production. + if isinstance(video_input, str): + try: + import decord as _decord + from decord import VideoReader as _VideoReader # type: ignore + + vr = _VideoReader(video_input, ctx=_decord.cpu(0)) + origin_fps = float(vr.get_avg_fps()) or origin_fps_default + idx_all = list(range(len(vr))) + video_raw = vr.get_batch(idx_all).asnumpy() # (T, H, W, 3) uint8 RGB + except ImportError: + import cv2 as _cv2 + + cap = _cv2.VideoCapture(video_input) + origin_fps = float(cap.get(_cv2.CAP_PROP_FPS) or origin_fps_default) + frames_bgr = [] + while True: + ok, f = cap.read() + if not ok: + break + frames_bgr.append(_cv2.cvtColor(f, _cv2.COLOR_BGR2RGB)) + cap.release() + video_raw = _np.stack(frames_bgr, axis=0) + elif isinstance(video_input, _np.ndarray): + video_raw = video_input + origin_fps = origin_fps_default + elif torch.is_tensor(video_input): + arr = video_input.detach().cpu().numpy() + if arr.max() <= 1.5: + arr = (arr * 255.0).clip(0, 255).astype(_np.uint8) + video_raw = arr + origin_fps = origin_fps_default + else: + raise ValueError(f"Unsupported video_input type {type(video_input)}") + cfg_text_scale = float(extra_args.get("cfg_text_scale", 4.0)) - cfg_img_scale = float(extra_args.get("cfg_img_scale", 1.5)) + cfg_img_scale = float(extra_args.get("cfg_img_scale", 1.0)) + # Upstream Lance default is ``cfg_interval=[0, 1]`` (CFG active on every + # denoise step). vllm-omni's previous (0.4, 1.0) skipped CFG for the + # last ~10 iterations after timestep_shift=3.5, which contributed + # most of the late-trajectory drift from upstream. + cfg_interval = tuple(extra_args.get("cfg_interval", (0.0, 1.0))) + cfg_renorm_type = str(extra_args.get("cfg_renorm_type", "global")) + cfg_renorm_min = float(extra_args.get("cfg_renorm_min", 0.0)) timestep_shift = float(extra_args.get("timestep_shift", LANCE_DEFAULTS.timestep_shift)) num_timesteps = int(req.sampling_params.num_inference_steps or LANCE_DEFAULTS.num_timesteps) - video_shape = (T, H, W) - logger.info("Lance video_edit: video %dx%dx%d (T,H,W)", T, H, W) - def vae_transforms(_): - return video_chw + # Run upstream's BucketResize + frame sampler so VAE / ViT see + # byte-identical pixels to the reference implementation. Without this, + # vllm-omni's ViT grid was (T=23, H=40, W=54) for the car video versus + # upstream's (T=23, H=36, W=50) — different K/V at ViT positions + # accounted for the residual ~9% rel_l2 at layer 0 output. + from .lance_transformer import LanceBagel as _LanceBagel + + vae_video, vit_pixels, vit_grid_thw, sampled_T = _LanceBagel._lance_video_preprocess(video_raw, origin_fps) + video_chw = vae_video # (C, T_sampled, H_vae, W_vae) — model-space [-1,1] + T_sampled = int(video_chw.shape[1]) + H_vae = int(video_chw.shape[2]) + W_vae = int(video_chw.shape[3]) + video_shape = (T_sampled, H_vae, W_vae) + logger.info( + "Lance video_edit: input %dx%dx%d → sampled %dx%dx%d (T,H,W) @ %.2ffps, ViT grid_thw=%s, user_text=%r", + video_raw.shape[0], + video_raw.shape[1], + video_raw.shape[2], + T_sampled, + H_vae, + W_vae, + origin_fps, + tuple(vit_grid_thw[0].tolist()), + user_text, + ) if req.sampling_params.seed is not None: torch.manual_seed(req.sampling_params.seed) if self.device.type == "cuda": torch.cuda.manual_seed(req.sampling_params.seed) - # Position-id layout for video editing — matches upstream Lance's - # ``shift_position_ids(pos_shift=1000)`` + ``modality1=modality2`` - # convention: - # * input video VAE block + output gen video latent share the same - # rope range [0, max(T_lat, H_lat, W_lat)] so the model can align - # "edit *this* token" between reference and output; - # * text + ViT prefill use a far-shifted rope start (``_EDIT_SHIFT``) - # so their positions never collide with the VAE/gen range. - _EDIT_SHIFT = 10000 gen_context = { "kv_lens": [0], "ropes": [0], "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), } - - # Multi-frame VAE prefill — rope starts at 0 (aligned with gen latent below). - vae_input, newlens_vae, _ = self.bagel._lance_native_prepare_vae_images( - curr_kvlens=gen_context["kv_lens"], - curr_rope=gen_context["ropes"], - images=[video_chw], - transforms=vae_transforms, - new_token_ids=self.new_token_ids, - is_video=True, - ) - for k, v in vae_input.items(): - if torch.is_tensor(v): - vae_input[k] = v.to(self.device) - with torch.autocast( - device_type=self.device.type, - enabled=self.device.type != "cpu", - dtype=self.od_config.dtype, - ): - gen_context["past_key_values"] = self.bagel.forward_cache_update_vae( - self.vae, gen_context["past_key_values"], **vae_input - ) - gen_context["kv_lens"] = newlens_vae - # Reset rope to the post-VAE shifted range for the text / ViT prefill; - # we'll snap back to 0 when constructing the gen latent so it shares - # positions with the VAE-prefilled reference. - gen_context["ropes"] = [_EDIT_SHIFT] - - # Multi-frame ViT prefill (reuse x2t_video prep). - vit_input, newlens_vit, new_rope_vit = self.bagel.prepare_vit_videos( - curr_kvlens=gen_context["kv_lens"], - curr_rope=gen_context["ropes"], - videos=[video_input], - new_token_ids=self.new_token_ids, - ) - for k, v in vit_input.items(): - if torch.is_tensor(v): - vit_input[k] = v.to(self.device) - with torch.autocast( - device_type=self.device.type, - enabled=self.device.type != "cpu", - dtype=self.od_config.dtype, - ): - gen_context["past_key_values"] = self.bagel.forward_cache_update_vit( - gen_context["past_key_values"], **vit_input - ) - gen_context["kv_lens"] = newlens_vit - gen_context["ropes"] = new_rope_vit - - # Text prefill. - text_input, newlens_text, new_rope_text = self.bagel.prepare_prompts( - curr_kvlens=gen_context["kv_lens"], - curr_rope=gen_context["ropes"], - prompts=[prompt], - tokenizer=self.tokenizer, - new_token_ids=self.new_token_ids, - ) - for k, v in text_input.items(): - if torch.is_tensor(v): - text_input[k] = v.to(self.device) - with torch.autocast( - device_type=self.device.type, - enabled=self.device.type != "cpu", - dtype=self.od_config.dtype, - ): - gen_context["past_key_values"] = self.bagel.forward_cache_update_text( - gen_context["past_key_values"], **text_input - ) - gen_context["kv_lens"] = newlens_text - gen_context["ropes"] = new_rope_text - - # CFG text branch. cfg_text_context = { "kv_lens": [0], "ropes": [0], "past_key_values": NaiveCache(self.bagel.config.llm_config.num_hidden_layers), } - if cfg_text_scale > 1.0: - neg_input, neg_newlens, neg_rope = self.bagel.prepare_prompts( - curr_kvlens=cfg_text_context["kv_lens"], - curr_rope=cfg_text_context["ropes"], - prompts=[str(extra_args.get("negative_prompt") or "")], - tokenizer=self.tokenizer, - new_token_ids=self.new_token_ids, - ) - for k, v in neg_input.items(): - if torch.is_tensor(v): - neg_input[k] = v.to(self.device) - with torch.autocast( - device_type=self.device.type, - enabled=self.device.type != "cpu", - dtype=self.od_config.dtype, - ): - cfg_text_context["past_key_values"] = self.bagel.forward_cache_update_text( - cfg_text_context["past_key_values"], **neg_input - ) - cfg_text_context["kv_lens"] = neg_newlens - cfg_text_context["ropes"] = neg_rope - # 3-D latent init (video output). ``curr_rope=[0]`` so the gen latent - # tokens share positions with the input VAE block above — that's what - # tells the model "edit token (t,h,w) in the input is token (t,h,w) in - # the output". The kv-cache still contains the text/ViT blocks at the - # shifted rope range, so attending over them works. + autocast_kwargs = dict( + device_type=self.device.type, + enabled=self.device.type != "cpu", + dtype=self.od_config.dtype, + ) + + def _raw_text_prefill(ctx, text_str): + """Prefill ``text_str`` as raw token ids — no bos/eos wrapping.""" + text_ids = self.tokenizer.encode(text_str, add_special_tokens=False) + if not text_ids: + return + curr_kvlen = ctx["kv_lens"][0] + curr_rope = ctx["ropes"][0] + seq_len = len(text_ids) + inp = { + "text_token_lens": torch.tensor([seq_len], dtype=torch.int, device=self.device), + "packed_text_ids": torch.tensor(text_ids, dtype=torch.long, device=self.device), + "packed_text_position_ids": torch.arange( + curr_rope, curr_rope + seq_len, dtype=torch.long, device=self.device + ), + "packed_text_indexes": torch.arange( + curr_kvlen, curr_kvlen + seq_len, dtype=torch.long, device=self.device + ), + "packed_key_value_indexes": torch.arange(0, curr_kvlen, dtype=torch.long, device=self.device), + "key_values_lens": torch.tensor([curr_kvlen], dtype=torch.int, device=self.device), + } + with torch.autocast(**autocast_kwargs): + ctx["past_key_values"] = self.bagel.forward_cache_update_text(ctx["past_key_values"], **inp) + ctx["kv_lens"] = [curr_kvlen + seq_len] + ctx["ropes"] = [curr_rope + seq_len] + + def _vit_prefill(ctx): + # Feed the bucket-resized + patchified ViT pixels directly so the + # Qwen2VLImageProcessor path (smart-resize, wrong grid) is bypassed. + inp, new_kvlens, new_rope = self.bagel.prepare_vit_videos( + curr_kvlens=ctx["kv_lens"], + curr_rope=ctx["ropes"], + videos=[video_raw], # passed through for length/iter only + new_token_ids=self.new_token_ids, + precomputed_vit=[(vit_pixels, vit_grid_thw)], + ) + for k, v in inp.items(): + if torch.is_tensor(v): + inp[k] = v.to(self.device) + with torch.autocast(**autocast_kwargs): + ctx["past_key_values"] = self.bagel.forward_cache_update_vit(ctx["past_key_values"], **inp) + ctx["kv_lens"] = new_kvlens + ctx["ropes"] = new_rope + + # Encode the reference video ONCE so both gen and cfg branches share + # the same posterior sample (see image_edit fix — Wan2.2 VAE's + # ``mu + std * randn_like(std)`` would otherwise drift between + # branches). + _ref_padded_latent_cache = {"v": None} + + def _vae_transforms(_): + return video_chw + + def _vae_prefill(ctx): + inp, new_kvlens, new_rope = self.bagel._lance_native_prepare_vae_images( + curr_kvlens=ctx["kv_lens"], + curr_rope=ctx["ropes"], + images=[video_chw], + transforms=_vae_transforms, + new_token_ids=self.new_token_ids, + is_video=True, + ) + for k, v in inp.items(): + if torch.is_tensor(v): + inp[k] = v.to(self.device) + if _ref_padded_latent_cache["v"] is None: + _ref_padded_latent_cache["v"] = self.vae.encode(inp["padded_images"].to(self.device)) + inp["precomputed_latent"] = _ref_padded_latent_cache["v"] + with torch.autocast(**autocast_kwargs): + ctx["past_key_values"] = self.bagel.forward_cache_update_vae(self.vae, ctx["past_key_values"], **inp) + ctx["kv_lens"] = new_kvlens + ctx["ropes"] = new_rope + + # ----- prefill segments in upstream order ----- + sys_prompt = SYSTEM_PROMPTS[("video_edit", "video")] + seg1_str = f"<|im_start|>system\n{sys_prompt}<|im_end|>\n<|im_start|>user\n" + seg5_str = "<|im_end|>\n<|im_start|>assistant\n" + + # seg1: system + user header (shared) + _raw_text_prefill(gen_context, seg1_str) + if cfg_text_scale > 1.0: + _raw_text_prefill(cfg_text_context, seg1_str) + + # seg2: ViT(ref) (shared) + _vit_prefill(gen_context) + if cfg_text_scale > 1.0: + _vit_prefill(cfg_text_context) + + # seg3: VAE(ref) — snapshot rope BEFORE the prefill so the gen noise + # latent can be placed at the SAME rope range (matches upstream's + # modality==1≡2 trick). + rope_before_vae = gen_context["ropes"][0] + cfg_rope_before_vae = cfg_text_context["ropes"][0] if cfg_text_scale > 1.0 else rope_before_vae + _vae_prefill(gen_context) + if cfg_text_scale > 1.0: + _vae_prefill(cfg_text_context) + + # seg4: user instruction (gen only — cfg branch skips per upstream). + _raw_text_prefill(gen_context, user_text) + + # seg5: separator + assistant header (both gen and cfg, each at its + # own current rope; cfg's lands earlier than gen's because cfg + # skipped seg4). + _raw_text_prefill(gen_context, seg5_str) + if cfg_text_scale > 1.0: + _raw_text_prefill(cfg_text_context, seg5_str) + + # seg6 (noise QUERY): gen latent at the SAME rope as the ref VAE + # block. KV cache still contains sys+ViT+VAE_ref+user_text+sep at + # their post-prefill rope positions; only the noise query's Q rope + # is rewound to overlap with the VAE_ref block. gen_input_lat = self.bagel.prepare_video_latent( curr_kvlens=gen_context["kv_lens"], - curr_rope=[0], + curr_rope=[rope_before_vae], video_shapes=[video_shape], new_token_ids=self.new_token_ids, ) @@ -1062,34 +1287,31 @@ class LancePipeline(BagelPipeline): gen_input_lat[k] = v.to(self.device) cfg_text_lat = self.bagel.prepare_video_latent_cfg( curr_kvlens=cfg_text_context["kv_lens"], - curr_rope=[0], - video_shapes=[video_shape], - ) - cfg_img_lat = self.bagel.prepare_video_latent_cfg( - curr_kvlens=gen_context["kv_lens"], - curr_rope=[0], + curr_rope=[cfg_rope_before_vae], video_shapes=[video_shape], ) for k, v in cfg_text_lat.items(): if torch.is_tensor(v): cfg_text_lat[k] = v.to(self.device) - for k, v in cfg_img_lat.items(): - if torch.is_tensor(v): - cfg_img_lat[k] = v.to(self.device) - with torch.autocast( - device_type=self.device.type, - enabled=self.device.type != "cpu", - dtype=self.od_config.dtype, - ): + # cfg_img branch is off (cfg_img_scale=1.0); generate_image needs the + # arg surface but skips the branch when scale<=1.0. + cfg_img_lat = cfg_text_lat + + with torch.autocast(**autocast_kwargs): latents, *_ = self.bagel.generate_image( past_key_values=gen_context["past_key_values"], - cfg_text_past_key_values=cfg_text_context["past_key_values"], + cfg_text_past_key_values=cfg_text_context["past_key_values"] + if cfg_text_scale > 1.0 + else gen_context["past_key_values"], cfg_img_past_key_values=gen_context["past_key_values"], num_timesteps=num_timesteps, timestep_shift=timestep_shift, cfg_text_scale=cfg_text_scale, cfg_img_scale=cfg_img_scale, + cfg_interval=cfg_interval, + cfg_renorm_type=cfg_renorm_type, + cfg_renorm_min=cfg_renorm_min, **gen_input_lat, cfg_text_packed_position_ids=cfg_text_lat["cfg_packed_position_ids"], cfg_text_packed_query_indexes=cfg_text_lat["cfg_packed_query_indexes"],