[Model] Add LongCat-Video-Avatar-1.5 ai2v, at2v support (#4099)

Signed-off-by: weiyan <weiyanlin117@gmail.com>
Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>

docs/models/supported_models.md

@@ -51,6 +51,7 @@ th {
 | `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `LongCatVideoAvatarPipeline` | LongCat-Video-Avatar-1.5 A2V/AI2V (native single-speaker and multi-speaker AI2V/AVC) | `meituan-longcat/LongCat-Video-Avatar-1.5` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/meituan-longcat/LongCat-Video-Avatar-1.5.md) |
 | `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit` | ✅︎ | | | | — |
 | `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | — |

docs/user_guide/diffusion_features.md

@@ -159,6 +159,7 @@ The following tables show which models support each feature:
 | **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
 | **DreamID-Omni**             |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **Cosmos3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
+| **LongCat-Video-Avatar-1.5** |     ❌     |     ❌      |           ❌           |       ❌        |         ❌         |         ❌         |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **MiniMax-H3**               | ✅ (FL2VA) |     ✅      |           ✅           |       ❌        |       ✅ (DiT/TE)  |         ❌         |   ✅    |             ✅             |       ✅ (tile)       |      ✅ (DiT)      |        ❌         |
 
 **Frame Interpolation Support**

examples/offline_inference/longcat_video/end2end.py

@@ -0,0 +1,319 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import argparse
+import json
+import re
+import subprocess
+import tempfile
+import wave
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+import imageio
+import numpy as np
+import torch
+
+from vllm_omni.diffusion.data import DiffusionParallelConfig
+from vllm_omni.diffusion.models.longcat_video.pipeline_longcat_video_avatar import (
+    prepare_longcat_video_avatar_model_for_omni,
+)
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+from vllm_omni.outputs import OmniRequestOutput
+from vllm_omni.platforms import current_omni_platform
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Generate LongCat-Video-Avatar A2V/AI2V video.")
+    parser.add_argument("--model", default="meituan-longcat/LongCat-Video-Avatar-1.5")
+    parser.add_argument("--base-model-dir", default=None, help="Optional local LongCat-Video base model directory.")
+    parser.add_argument("--stage", choices=["at2v", "ai2v"], default="at2v")
+    parser.add_argument("--prompt", default="A person speaks calmly while facing the camera.")
+    parser.add_argument("--negative-prompt", default="low quality, blurry, watermark, text")
+    parser.add_argument("--audio", default=None, help="Input speech audio path.")
+    parser.add_argument("--image", default=None, help="Input reference image path for AI2V.")
+    parser.add_argument("--input-json", default=None, help="Official LongCat Avatar JSON case for AI2V.")
+    parser.add_argument("--output", default="longcat_avatar_output.mp4")
+    parser.add_argument("--resolution", choices=["480p", "720p"], default="480p")
+    parser.add_argument("--num-frames", type=int, default=93)
+    parser.add_argument("--num-segments", default="1", help="Number of AVC segments, or 'auto' to cover full audio.")
+    parser.add_argument("--num-cond-frames", type=int, default=13)
+    parser.add_argument("--ref-img-index", type=int, default=10)
+    parser.add_argument("--mask-frame-range", type=int, default=3)
+    parser.add_argument("--fps", type=int, default=25)
+    parser.add_argument("--num-inference-steps", type=int, default=8)
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--use-distill", action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument("--use-int8", action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument(
+        "--build-components-on-gpu",
+        action="store_true",
+        default=False,
+        help="Build large Avatar components directly on GPU for faster startup. Requires more peak VRAM.",
+    )
+    parser.add_argument("--use-kv-cache", action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument("--offload-kv-cache", action=argparse.BooleanOptionalAction, default=False)
+    parser.add_argument("--enforce-eager", action="store_true")
+    return parser.parse_args()
+
+
+def _extract_frames(output: Any):
+    if isinstance(output, list):
+        output = output[0] if output else None
+    if isinstance(output, OmniRequestOutput):
+        if not output.images:
+            raise ValueError("No video frames found in OmniRequestOutput.")
+        frames = output.images[0] if len(output.images) == 1 else output.images
+        if isinstance(frames, tuple) and len(frames) == 2:
+            frames = frames[0]
+        if isinstance(frames, dict):
+            frames = frames.get("frames") or frames.get("video")
+    else:
+        frames = output
+    if frames is None:
+        raise ValueError("No video frames found in output.")
+    return frames
+
+
+def _frames_to_uint8(frames):
+    frames_np = np.asarray([np.asarray(frame) for frame in frames])
+    if frames_np.ndim == 5:
+        frames_np = frames_np[0]
+    if np.issubdtype(frames_np.dtype, np.floating):
+        frames_np = (np.clip(frames_np, 0.0, 1.0) * 255).round()
+    return np.clip(frames_np, 0, 255).astype(np.uint8)
+
+
+@lru_cache(maxsize=1)
+def _ffmpeg_exe() -> str:
+    try:
+        import imageio_ffmpeg
+
+        return imageio_ffmpeg.get_ffmpeg_exe()
+    except Exception as exc:
+        raise RuntimeError("LongCat Avatar example requires the Python imageio[ffmpeg] dependency.") from exc
+
+
+def _wav_duration(path: Path) -> float:
+    with wave.open(str(path), "rb") as audio:
+        return audio.getnframes() / float(audio.getframerate())
+
+
+def _save_video_with_audio(frames, output_path: Path, audio_path: str, fps: int) -> None:
+    frames_np = _frames_to_uint8(frames)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix="longcat_avatar_export_") as tmp:
+        tmp_dir = Path(tmp)
+        temp_video = tmp_dir / "video.mp4"
+        crop_audio = tmp_dir / "audio.wav"
+        crop_video = tmp_dir / "video_cropped.mp4"
+        writer = imageio.get_writer(str(temp_video), fps=fps, quality=5)
+        try:
+            for frame in frames_np:
+                writer.append_data(frame)
+        finally:
+            writer.close()
+
+        duration = len(frames_np) / fps
+        ffmpeg = _ffmpeg_exe()
+        subprocess.run([ffmpeg, "-y", "-i", audio_path, "-t", f"{duration}", str(crop_audio)], check=True)
+        audio_duration = _wav_duration(crop_audio)
+        subprocess.run(
+            [
+                ffmpeg,
+                "-y",
+                "-i",
+                str(temp_video),
+                "-t",
+                f"{audio_duration}",
+                "-c:v",
+                "copy",
+                str(crop_video),
+            ],
+            check=True,
+        )
+        subprocess.run(
+            [
+                ffmpeg,
+                "-y",
+                "-i",
+                str(crop_video),
+                "-i",
+                str(crop_audio),
+                "-c:v",
+                "libx264",
+                "-c:a",
+                "aac",
+                "-shortest",
+                str(output_path),
+            ],
+            check=True,
+        )
+
+
+def _infer_asset_root_from_input_json(input_json: Path) -> Path:
+    resolved = input_json.resolve()
+    parts = resolved.parts
+    if "assets" not in parts:
+        return resolved.parent
+    asset_idx = parts.index("assets")
+    return Path(*parts[:asset_idx])
+
+
+def _resolve_case_path(path: str, asset_root: Path) -> str:
+    raw_path = Path(path)
+    if raw_path.is_absolute():
+        return str(raw_path)
+    candidate = asset_root / raw_path
+    return str(candidate if candidate.exists() else raw_path)
+
+
+def _speaker_sort_key(name: str) -> tuple[int, str]:
+    match = re.search(r"(\d+)$", str(name))
+    return (int(match.group(1)) if match else 10_000, str(name))
+
+
+def _load_input_case(input_json: str) -> tuple[dict[str, Any], Path]:
+    input_json_path = Path(input_json)
+    with input_json_path.open("r", encoding="utf-8") as f:
+        case = json.load(f)
+    return case, _infer_asset_root_from_input_json(input_json_path)
+
+
+def _create_merged_audio_for_case(
+    audio_paths: dict[str, str],
+    audio_type: str,
+    output_path: Path,
+) -> str:
+    sorted_paths = [path for _, path in sorted(audio_paths.items(), key=lambda item: _speaker_sort_key(str(item[0])))]
+    if len(sorted_paths) == 1:
+        return sorted_paths[0]
+    inputs = [arg for path in sorted_paths for arg in ("-i", path)]
+    audio_type = audio_type.lower()
+    if audio_type == "para":
+        filter_complex = f"amix=inputs={len(sorted_paths)}:duration=longest:normalize=0"
+    elif audio_type == "add":
+        filter_complex = "".join(f"[{idx}:a]" for idx in range(len(sorted_paths)))
+        filter_complex += f"concat=n={len(sorted_paths)}:v=0:a=1"
+    else:
+        raise NotImplementedError(f"Unsupported LongCat Avatar audio_type {audio_type!r}.")
+    subprocess.run([_ffmpeg_exe(), "-y", *inputs, "-filter_complex", filter_complex, str(output_path)], check=True)
+    return str(output_path)
+
+
+def main():
+    args = parse_args()
+    input_case = None
+    input_asset_root = None
+    audio_for_mux = args.audio
+    if args.input_json:
+        input_case, input_asset_root = _load_input_case(args.input_json)
+        args.stage = "ai2v"
+        args.prompt = input_case.get("prompt") or args.prompt
+        args.image = _resolve_case_path(input_case["cond_image"], input_asset_root)
+        args.audio = None
+    if not args.input_json and not args.audio:
+        raise ValueError("--audio is required unless --input-json is provided.")
+    if args.stage == "ai2v" and not args.image:
+        raise ValueError("--image is required when --stage=ai2v.")
+    model_for_omni = prepare_longcat_video_avatar_model_for_omni(args.model, args.use_int8)
+
+    additional_config = {
+        "model_type": "avatar-v1.5",
+        "resolution": args.resolution,
+        "use_distill": args.use_distill,
+        "use_int8": args.use_int8,
+        "build_components_on_gpu": args.build_components_on_gpu,
+    }
+    if args.base_model_dir:
+        additional_config["base_model_dir"] = args.base_model_dir
+
+    omni = Omni(
+        model=model_for_omni,
+        model_class_name="LongCatVideoAvatarPipeline",
+        dtype="bfloat16",
+        enforce_eager=args.enforce_eager,
+        additional_config=additional_config,
+        parallel_config=DiffusionParallelConfig(
+            ulysses_degree=1,
+            ring_degree=1,
+            cfg_parallel_size=1,
+            tensor_parallel_size=1,
+            vae_patch_parallel_size=1,
+            pipeline_parallel_size=1,
+        ),
+    )
+
+    audio_input: str | dict[str, str] | None = args.audio
+    additional_information = {"stage": args.stage, "resolution": args.resolution}
+    if input_case is not None and input_asset_root is not None:
+        audio_input = {
+            speaker: _resolve_case_path(path, input_asset_root)
+            for speaker, path in sorted(
+                input_case["cond_audio"].items(),
+                key=lambda item: _speaker_sort_key(str(item[0])),
+            )
+        }
+        additional_information.update(
+            {
+                "input_json": args.input_json,
+                "audio_type": input_case.get("audio_type", "para"),
+                "bbox": input_case.get("bbox", {}),
+            }
+        )
+
+    prompt = {
+        "prompt": args.prompt,
+        "negative_prompt": args.negative_prompt,
+        "multi_modal_data": {"audio": audio_input},
+        "additional_information": additional_information,
+    }
+    if args.stage == "ai2v":
+        prompt["multi_modal_data"]["image"] = args.image
+
+    try:
+        output = omni.generate(
+            prompt,
+            OmniDiffusionSamplingParams(
+                generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed),
+                seed=args.seed,
+                guidance_scale=1.0 if args.use_distill else 4.0,
+                guidance_scale_2=1.0 if args.use_distill else 4.0,
+                num_inference_steps=args.num_inference_steps,
+                num_frames=args.num_frames,
+                fps=args.fps,
+                extra_args={
+                    "stage": args.stage,
+                    "resolution": args.resolution,
+                    "save_fps": args.fps,
+                    "use_distill": args.use_distill,
+                    "use_int8": args.use_int8,
+                    "input_json": args.input_json,
+                    "num_segments": args.num_segments,
+                    "num_cond_frames": args.num_cond_frames,
+                    "ref_img_index": args.ref_img_index,
+                    "mask_frame_range": args.mask_frame_range,
+                    "use_kv_cache": args.use_kv_cache,
+                    "offload_kv_cache": args.offload_kv_cache,
+                },
+            ),
+            use_tqdm=True,
+        )
+    finally:
+        omni.close()
+
+    with tempfile.TemporaryDirectory(prefix="longcat_avatar_audio_mux_") as tmp:
+        if input_case is not None and isinstance(audio_input, dict):
+            audio_for_mux = _create_merged_audio_for_case(
+                audio_input,
+                input_case.get("audio_type", "para"),
+                Path(tmp) / "merged_audio.wav",
+            )
+        _save_video_with_audio(_extract_frames(output), Path(args.output), audio_for_mux, args.fps)
+    print(f"Saved video to {args.output}")
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -89,6 +89,14 @@ local = [
     "FlagEmbedding",
 ]
 
+# LongCat-Video-Avatar A2V/AI2V runtime dependencies for vocal
+# separation and loudness normalization.
+longcat-video-avatar = [
+    "audio-separator==0.44.2",
+    "pyloudnorm==0.1.1",
+    "soundfile>=0.13.1",
+]
+
 # Optional Blackwell-only fused-bias FP8 GEMM; package is `quack-kernels` (imported
 # as `quack`), not the unrelated `quack`. See docs/user_guide/quantization/fp8.md.
 quack = [

recipes/README.md

@@ -55,6 +55,7 @@ recipes/
 | [`krea/Krea-2.md`](./krea/Krea-2.md) | Text-to-image (Turbo + Raw), offline + online, with LoRA | 1x H100 80GB |
 | [`LTX/LTX-2.md`](./LTX/LTX-2.md) | LTX-2/LTX-2.3 text-to-video and image-to-video with synchronized audio | H200 141GB / 96GB-class GPU |
 | [`MammothModa2/MammothModa2.md`](./MammothModa2/MammothModa2.md) | Preview and Dev text-to-image (AR → DiT); Dev text/image understanding | Preview: 1x L40S 48GB / 1x ≥40GB GPU; Dev: 1x NVIDIA GPU with sufficient cache headroom |
+| [`meituan-longcat/LongCat-Video-Avatar-1.5.md`](./meituan-longcat/LongCat-Video-Avatar-1.5.md) | Audio-driven avatar video generation (AT2V / AI2V, single- and multi-speaker, AVC continuation) | 1x H100 80GB |
 | [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB |
 | [`Robbyant/LingBot-Video.md`](./Robbyant/LingBot-Video.md) | Native dense and MoE text-to-video serving | Dense: 1x L20X; MoE: 1x L20X (~67.7 GiB peak) |
 | [`cosmos3/Cosmos3-Nano.md`](./cosmos3/Cosmos3-Nano.md) | Text-to-image, text-to-video, image-to-video, video-to-video generation, text to video with sound, action policy | 1x H200 141GB / B300 |

recipes/meituan-longcat/LongCat-Video-Avatar-1.5.md

@@ -0,0 +1,282 @@
+# LongCat-Video-Avatar-1.5
+
+> Audio-driven avatar video generation (AT2V / AI2V)
+
+## Summary
+
+- Vendor: meituan-longcat
+- Model: `meituan-longcat/LongCat-Video-Avatar-1.5`
+- Task: Audio-to-video (AT2V) and audio-and-image-to-video (AI2V) avatar generation
+- Mode: Offline inference
+- Maintainer: Community
+
+## When to use this recipe
+
+Use this recipe to drive a talking or singing avatar from a speech clip, either
+from text alone (AT2V) or from a reference image (AI2V), with the native
+LongCat-Video-Avatar-1.5 pipeline in vLLM-Omni. It covers:
+
+1. **Single-speaker AT2V / AI2V** — one 93-frame segment from one audio track.
+2. **AVC continuation** — chained segments that cover a longer audio track.
+3. **Multi-speaker AI2V** — several audio tracks placed on a reference image
+   through speaker bounding boxes.
+
+The example saves an mp4 with the generated frames muxed with the input audio.
+
+## References
+
+- Upstream model card: <https://huggingface.co/meituan-longcat/LongCat-Video-Avatar-1.5>
+- Upstream repository and example assets: <https://github.com/meituan-longcat/LongCat-Video>
+- Related example under `examples/`: [`longcat_video/end2end.py`](../../examples/offline_inference/longcat_video/end2end.py)
+
+## Hardware Support
+
+## CUDA
+
+### 1× NVIDIA H100 (80 GB)
+
+#### Environment
+
+- OS: Linux
+- Python: 3.10+
+- Driver: NVIDIA driver with CUDA 12.x
+- vLLM version: Match the repository requirements for your checkout
+- vLLM-Omni version or commit: Use the commit you are deploying from
+
+#### Prerequisites
+
+Install the LongCat Avatar optional runtime dependencies:
+
+```bash
+pip install "vllm-omni[longcat-video-avatar]"
+```
+
+The example uses the `imageio_ffmpeg` executable installed by vLLM-Omni's
+`imageio[ffmpeg]` dependency.
+
+#### Command
+
+Audio-to-video (AT2V):
+
+```bash
+python examples/offline_inference/longcat_video/end2end.py \
+  --model meituan-longcat/LongCat-Video-Avatar-1.5 \
+  --stage at2v \
+  --audio /path/to/speech.wav \
+  --prompt "A person speaks calmly while facing the camera." \
+  --num-frames 93 \
+  --num-inference-steps 8 \
+  --output longcat_avatar_at2v.mp4
+```
+
+Audio-and-image-to-video (AI2V):
+
+```bash
+python examples/offline_inference/longcat_video/end2end.py \
+  --model meituan-longcat/LongCat-Video-Avatar-1.5 \
+  --stage ai2v \
+  --audio /path/to/speech.wav \
+  --image /path/to/reference.jpg \
+  --prompt "A person speaks calmly while facing the camera." \
+  --num-frames 93 \
+  --num-inference-steps 8 \
+  --output longcat_avatar_ai2v.mp4
+```
+
+#### Official Example Assets
+
+Download the official Avatar example assets:
+
+```bash
+mkdir -p longcat_avatar_assets/assets/avatar/single longcat_avatar_assets/assets/avatar/multi
+
+curl -L -o longcat_avatar_assets/assets/avatar/single/man.mp3 \
+  https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/avatar/single/man.mp3
+curl -L -o longcat_avatar_assets/assets/avatar/single/man.png \
+  https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/avatar/single/man.png
+curl -L -o longcat_avatar_assets/assets/avatar/single_example_1.json \
+  https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/avatar/single_example_1.json
+
+curl -L -o longcat_avatar_assets/assets/avatar/multi/sing.png \
+  https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/avatar/multi/sing.png
+curl -L -o longcat_avatar_assets/assets/avatar/multi/sing_man.WAV \
+  https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/avatar/multi/sing_man.WAV
+curl -L -o longcat_avatar_assets/assets/avatar/multi/sing_woman.WAV \
+  https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/avatar/multi/sing_woman.WAV
+curl -L -o longcat_avatar_assets/assets/avatar/multi/introduce.png \
+  https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/avatar/multi/introduce.png
+curl -L -o longcat_avatar_assets/assets/avatar/multi/introduce_man.mp3 \
+  https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/avatar/multi/introduce_man.mp3
+curl -L -o longcat_avatar_assets/assets/avatar/multi/introduce_woman.mp3 \
+  https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/avatar/multi/introduce_woman.mp3
+curl -L -o longcat_avatar_assets/assets/avatar/multi_example_1.json \
+  https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/avatar/multi_example_1.json
+curl -L -o longcat_avatar_assets/assets/avatar/multi_example_2.json \
+  https://raw.githubusercontent.com/meituan-longcat/LongCat-Video/main/assets/avatar/multi_example_2.json
+
+export LONGCAT_VIDEO_ASSET_DIR="$PWD/longcat_avatar_assets/assets/avatar"
+export AVATAR_MODEL=meituan-longcat/LongCat-Video-Avatar-1.5
+export PROMPT="$(python -c 'import json, os; print(json.load(open(os.environ["LONGCAT_VIDEO_ASSET_DIR"] + "/single_example_1.json"))["prompt"])')"
+```
+
+`--base-model-dir` points at a local LongCat-Video base checkpoint for the
+tokenizer, text encoder and VAE; omit it to download those components from
+Hugging Face.
+
+```bash
+export BASE_MODEL_DIR=/cache/longcat_models/LongCat-Video
+```
+
+#### Official Asset Smoke Test
+
+Audio-to-video with the official audio:
+
+```bash
+python examples/offline_inference/longcat_video/end2end.py \
+  --model "$AVATAR_MODEL" \
+  --base-model-dir "$BASE_MODEL_DIR" \
+  --stage at2v \
+  --audio "$LONGCAT_VIDEO_ASSET_DIR/single/man.mp3" \
+  --prompt "$PROMPT" \
+  --num-frames 93 \
+  --num-inference-steps 8 \
+  --output official_asset_longcat_avatar_at2v.mp4
+```
+
+Audio-and-image-to-video with the official audio and image:
+
+```bash
+python examples/offline_inference/longcat_video/end2end.py \
+  --model "$AVATAR_MODEL" \
+  --base-model-dir "$BASE_MODEL_DIR" \
+  --stage ai2v \
+  --audio "$LONGCAT_VIDEO_ASSET_DIR/single/man.mp3" \
+  --image "$LONGCAT_VIDEO_ASSET_DIR/single/man.png" \
+  --prompt "$PROMPT" \
+  --num-frames 93 \
+  --num-inference-steps 8 \
+  --output official_asset_longcat_avatar_ai2v.mp4
+```
+
+#### Official Single-Speaker AVC Continuation
+
+`--num-segments` chains AVC segments; `auto` covers the full input audio.
+
+```bash
+python examples/offline_inference/longcat_video/end2end.py \
+  --model "$AVATAR_MODEL" \
+  --base-model-dir "$BASE_MODEL_DIR" \
+  --input-json "$LONGCAT_VIDEO_ASSET_DIR/single_example_1.json" \
+  --num-frames 93 \
+  --num-cond-frames 13 \
+  --num-segments 5 \
+  --num-inference-steps 8 \
+  --output official_asset_longcat_avatar_single_example_1_ai2v_5seg.mp4
+```
+
+Replace `--num-segments 5` with `--num-segments auto` to run until the full
+official audio is covered. That is a slow archival validation path rather than a
+regular smoke test: about 24m28s on a Modal H100 for the full
+`single_example_1.json` audio with 8 inference steps.
+
+#### Official Multi-Speaker AI2V
+
+The multi-speaker cases use several audio tracks with speaker masks and optional
+bounding boxes. The command below generates the first 93-frame segment, matching
+the first call to the official `generate_ai2v()`.
+
+```bash
+python examples/offline_inference/longcat_video/end2end.py \
+  --model "$AVATAR_MODEL" \
+  --base-model-dir "$BASE_MODEL_DIR" \
+  --input-json "$LONGCAT_VIDEO_ASSET_DIR/multi_example_1.json" \
+  --num-frames 93 \
+  --num-inference-steps 8 \
+  --output official_asset_longcat_avatar_multi_example_1_ai2v.mp4
+```
+
+Use `multi_example_2.json` for the sequential-speaking case, and add
+`--num-cond-frames 13 --num-segments auto` to cover the full audio with AVC
+continuation.
+
+#### Verification
+
+```bash
+ffprobe -hide_banner longcat_avatar_ai2v.mp4
+```
+
+The output must report one video stream and one audio stream: the example muxes
+the generated frames with the input audio.
+
+#### Notes
+
+**Support matrix**
+
+| Mode | Speakers | Image required | AVC continuation |
+| --- | --- | --- | --- |
+| AT2V | Single | No | Yes |
+| AI2V | Single | Yes | Yes |
+| AT2V | Multi | Not supported | Not supported |
+| AI2V | Multi | Yes | Yes |
+
+Multi-speaker Avatar requires AI2V because speaker bounding boxes and masks are
+defined on a reference image; AT2V with several tracks fails with an explicit
+error. AVC continuation uses full-audio embeddings only when `--num-segments` is
+greater than `1` or set to `auto`; single-segment AT2V/AI2V keeps the shorter
+first-segment audio path.
+
+**Sampling parameters**
+
+| Parameter | LongCat pipeline value | Description |
+| --- | --- | --- |
+| `--num-frames` | `num_frames` | Frames generated per segment. The default is 93 frames, or about 3.72 seconds at 25 FPS. LongCat normalizes other values to the temporal-VAE-compatible `4k+1` form. |
+| `--num-inference-steps` | `steps` | Denoising steps. The distilled Avatar path uses 8 steps. |
+| `--fps` | `save_fps` | Output frame rate, and the rate used to align audio embeddings with video frames. The default is 25 FPS. |
+| `sampling_params.guidance_scale` | `text_guidance_scale` | Classifier-free guidance applied to the text prompt. |
+| `sampling_params.guidance_scale_2` | `audio_guidance_scale` | Classifier-free guidance applied to the speech audio. |
+
+With the distilled LoRA enabled (the default), the pipeline uses 8 inference
+steps and sets both guidance scales to `1.0`. With `--no-use-distill`, the
+example sets both to `4.0`. For AVC continuation every segment holds
+`num_frames` frames, but each segment after the first reuses `num_cond_frames`
+conditioning frames, so the unique frame count is
+`num_frames + (num_segments - 1) * (num_frames - num_cond_frames)`.
+
+**Key flags**
+
+- `--stage`: `at2v` for audio-to-video, `ai2v` for audio-and-image-to-video.
+- `--resolution`: `480p` or `720p`.
+- `--input-json`: official LongCat Avatar JSON case, used for the single-speaker
+  and multi-speaker AI2V examples.
+- `--num-segments`: number of AVC segments, or `auto` to cover the full audio.
+- `--num-cond-frames`: previous-frame conditioning window for AVC continuation.
+- `--use-int8` / `--no-use-int8`: load the official INT8 Avatar DiT weights by
+  default, or full precision weights.
+- `--use-distill` / `--no-use-distill`: enable the official distilled LoRA path
+  by default.
+- `--build-components-on-gpu`: build large Avatar components directly on GPU for
+  faster startup, at a higher peak VRAM cost. Disabled by default.
+- `--base-model-dir`: local LongCat-Video base model directory for the
+  tokenizer, text encoder and VAE components.
+
+**Memory usage**
+
+Measured on a Modal H100 running the official 93-frame AI2V example with the
+INT8 DiT, distilled LoRA and 8 inference steps:
+
+| Component build | CLI option | Peak GPU memory | Model loading time |
+| --- | --- | ---: | ---: |
+| CPU (default) | None | About 41.0 GiB | About 130s |
+| GPU | `--build-components-on-gpu` | About 56.8 GiB | About 33s |
+
+The reported peak covers the end-to-end run. Building components on CPU only
+reduces initialization memory; denoising still runs on GPU. These numbers are
+memory guidance rather than a performance benchmark.
+
+**Known limitations**
+
+- Single GPU only. Cache acceleration, sequence/CFG/tensor parallelism, HSDP,
+  CPU offload, VAE patch parallelism and quantization are not wired up for this
+  pipeline yet; see the diffusion feature matrix in
+  [`docs/user_guide/diffusion_features.md`](../../docs/user_guide/diffusion_features.md).
+- Offline inference only. Online serving is not supported yet.

tests/diffusion/models/longcat_video/test_longcat_video_avatar.py

@@ -0,0 +1,266 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from __future__ import annotations
+
+import json
+
+import numpy as np
+import pytest
+import torch
+from PIL import Image
+
+from vllm_omni.diffusion.models.longcat_video.longcat_video_avatar_transformer import (
+    LongCatVideoAvatarTransformer3DModel,
+    _read_config,
+    replace_linear_with_quantized,
+)
+from vllm_omni.diffusion.models.longcat_video.pipeline_longcat_video_avatar import (
+    _avatar_model_allow_patterns,
+    _build_multi_speaker_ref_target_masks,
+    _default_at2v_shape,
+    _infer_asset_root_from_path,
+    _prepare_multi_speaker_audio_arrays,
+    _resolve_num_segments,
+    prepare_longcat_video_avatar_model_for_omni,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
+
+
+@pytest.mark.parametrize(
+    ("resolution", "expected_shape"),
+    [
+        ("480p", (480, 832)),
+        ("720p", (736, 1248)),
+    ],
+)
+def test_longcat_video_avatar_at2v_default_shape_uses_resolution_bucket(
+    resolution: str,
+    expected_shape: tuple[int, int],
+):
+    assert _default_at2v_shape(resolution) == expected_shape
+
+
+@pytest.mark.parametrize(
+    ("use_int8", "expected_weight_dir", "unexpected_weight_dir"),
+    [
+        (True, "base_model_int8/*", "base_model/*"),
+        (False, "base_model/*", "base_model_int8/*"),
+    ],
+)
+def test_longcat_video_avatar_allow_patterns_download_one_weight_set(
+    use_int8: bool,
+    expected_weight_dir: str,
+    unexpected_weight_dir: str,
+):
+    allow_patterns = _avatar_model_allow_patterns(use_int8)
+
+    assert expected_weight_dir in allow_patterns
+    assert unexpected_weight_dir not in allow_patterns
+    assert "whisper-large-v3/*" not in allow_patterns
+    assert "whisper-large-v3/model.safetensors" in allow_patterns
+    assert "whisper-large-v3/pytorch_model.bin" not in allow_patterns
+    assert "whisper-large-v3/flax_model.msgpack" not in allow_patterns
+    assert "vocal_separator/*" in allow_patterns
+
+
+def test_longcat_video_avatar_prepare_model_adds_omni_metadata(tmp_path):
+    model_dir = tmp_path / "LongCat-Video-Avatar-1.5"
+    base_model_dir = model_dir / "base_model_int8"
+    base_model_dir.mkdir(parents=True)
+    (base_model_dir / "config.json").write_text('{"model_type": "longcat_avatar"}', encoding="utf-8")
+    (model_dir / "model_index.json").write_text("{}", encoding="utf-8")
+
+    prepared = prepare_longcat_video_avatar_model_for_omni(str(model_dir), use_int8=True)
+
+    assert prepared == str(model_dir)
+    model_index = json.loads((model_dir / "model_index.json").read_text(encoding="utf-8"))
+    assert model_index["_class_name"] == "LongCatVideoAvatarPipeline"
+    assert model_index["_diffusers_version"] == "0.38.0"
+    transformer_config = json.loads((model_dir / "transformer" / "config.json").read_text(encoding="utf-8"))
+    assert transformer_config == {"model_type": "longcat_avatar"}
+
+
+def test_longcat_video_avatar_infers_repo_root_from_official_asset_path(tmp_path):
+    input_json = tmp_path / "assets" / "avatar" / "single_example_1.json"
+    input_json.parent.mkdir(parents=True)
+    input_json.write_text("{}", encoding="utf-8")
+
+    assert _infer_asset_root_from_path(input_json) == tmp_path
+
+
+def test_longcat_video_avatar_falls_back_to_json_parent_without_official_layout(tmp_path):
+    input_json = tmp_path / "single_example_1.json"
+    input_json.write_text("{}", encoding="utf-8")
+
+    assert _infer_asset_root_from_path(input_json) == tmp_path
+
+
+@pytest.mark.parametrize(
+    ("value", "audio_duration", "expected"),
+    [
+        (None, 26.6, 1),
+        ("1", 26.6, 1),
+        (3, 26.6, 3),
+        ("auto", 3.0, 1),
+        ("auto", 23.8, 8),
+        ("auto", 26.6, 9),
+    ],
+)
+def test_longcat_video_avatar_resolve_num_segments(
+    value,
+    audio_duration: float,
+    expected: int,
+):
+    assert (
+        _resolve_num_segments(
+            value,
+            audio_duration=audio_duration,
+            num_frames=93,
+            num_cond_frames=13,
+            save_fps=25,
+        )
+        == expected
+    )
+
+
+def test_longcat_video_avatar_read_config_filters_non_constructor_metadata(tmp_path):
+    config_path = tmp_path / "config.json"
+    config_path.write_text(
+        json.dumps(
+            {
+                "_class_name": "LongCatVideoAvatarTransformer3DModel",
+                "architectures": ["LongCatVideoAvatarTransformer3DModel"],
+                "_diffusers_version": "0.35.1",
+                "model_max_length": 512,
+                "hidden_size": 4096,
+                "depth": 48,
+                "num_heads": 32,
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    config = _read_config(config_path)
+
+    assert config == {
+        "hidden_size": 4096,
+        "depth": 48,
+        "num_heads": 32,
+    }
+
+
+def test_longcat_video_avatar_load_weights_supports_int8_buffers():
+    model = LongCatVideoAvatarTransformer3DModel(
+        hidden_size=4,
+        depth=0,
+        num_heads=1,
+        caption_channels=4,
+        intermediate_dim=4,
+        output_dim=4,
+        audio_channel=4,
+        context_tokens=1,
+    )
+    replace_linear_with_quantized(model)
+
+    buffer_name = "t_embedder.mlp.0.weight_int8"
+    buffers = dict(model.named_buffers())
+    loaded_weight = torch.ones_like(buffers[buffer_name])
+
+    loaded_params = model.load_weights([(buffer_name, loaded_weight)])
+
+    assert buffer_name in loaded_params
+    assert torch.equal(dict(model.named_buffers())[buffer_name], loaded_weight)
+
+
+def test_longcat_video_avatar_multi_speaker_para_audio_arrays_are_aligned():
+    left = np.ones(3, dtype=np.float32)
+    right = np.ones(5, dtype=np.float32) * 2
+
+    left_out, right_out = _prepare_multi_speaker_audio_arrays(
+        [left, right],
+        audio_type="para",
+        generate_duration=0.5,
+        sample_rate=10,
+    )
+
+    assert left_out.tolist() == [1, 1, 1, 0, 0]
+    assert right_out.tolist() == [2, 2, 2, 2, 2]
+
+
+def test_longcat_video_avatar_multi_speaker_add_audio_arrays_are_sequential():
+    left = np.ones(3, dtype=np.float32)
+    right = np.ones(2, dtype=np.float32) * 2
+
+    left_out, right_out = _prepare_multi_speaker_audio_arrays(
+        [left, right],
+        audio_type="add",
+        generate_duration=0.1,
+        sample_rate=10,
+    )
+
+    assert left_out.tolist() == [1, 1, 1, 0, 0]
+    assert right_out.tolist() == [0, 0, 0, 2, 2]
+
+
+def test_longcat_video_avatar_multi_speaker_masks_use_explicit_bboxes():
+    image = Image.new("RGB", (8, 6))
+    masks, use_background = _build_multi_speaker_ref_target_masks(
+        image,
+        {
+            "person1": [1, 1, 4, 3],
+            "person2": [2, 4, 5, 7],
+        },
+    )
+
+    assert not use_background
+    assert masks.shape == (3, 6, 8)
+    assert masks[0, 1:4, 1:3].sum() == 6
+    assert masks[1, 2:5, 4:7].sum() == 9
+    assert masks[2, 0, 0] == 1
+    assert masks[2, 2, 2] == 0
+
+
+def test_longcat_video_avatar_multi_speaker_masks_default_to_left_right_split():
+    image = Image.new("RGB", (10, 8))
+    masks, use_background = _build_multi_speaker_ref_target_masks(image, None)
+
+    assert not use_background
+    assert masks.shape == (3, 8, 10)
+    assert masks[0, :, :5].sum() > 0
+    assert masks[0, :, 5:].sum() == 0
+    assert masks[1, :, :5].sum() == 0
+    assert masks[1, :, 5:].sum() > 0
+
+
+def test_longcat_video_avatar_transformer_accepts_multi_speaker_masks():
+    model = LongCatVideoAvatarTransformer3DModel(
+        hidden_size=4,
+        depth=0,
+        num_heads=1,
+        caption_channels=4,
+        intermediate_dim=4,
+        output_dim=4,
+        audio_channel=4,
+        context_tokens=1,
+    )
+    hidden_states = torch.randn(1, 16, 5, 4, 4)
+    timestep = torch.zeros(1, 5)
+    encoder_hidden_states = torch.randn(1, 1, 4, 4)
+    audio_embs = torch.randn(2, 5, 5, 12, 4)
+    ref_target_masks = torch.zeros(3, 4, 4)
+    ref_target_masks[0, :, :2] = 1
+    ref_target_masks[1, :, 2:] = 1
+    ref_target_masks[2] = torch.where(ref_target_masks[0] + ref_target_masks[1] > 0, 0, 1)
+
+    output = model(
+        hidden_states=hidden_states,
+        timestep=timestep,
+        encoder_hidden_states=encoder_hidden_states,
+        num_cond_latents=1,
+        audio_embs=audio_embs,
+        ref_target_masks=ref_target_masks,
+    )
+
+    assert output.shape == hidden_states.shape

tests/e2e/offline_inference/test_longcat_video_avatar.py

@@ -0,0 +1,246 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from collections.abc import Generator
+
+import numpy as np
+import pytest
+import soundfile as sf
+from PIL import Image
+
+from tests.helpers.mark import hardware_test
+from tests.helpers.runtime import OmniRunner
+from vllm_omni.diffusion.models.longcat_video.pipeline_longcat_video_avatar import (
+    prepare_longcat_video_avatar_model_for_omni,
+)
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+MODEL = "meituan-longcat/LongCat-Video-Avatar-1.5"
+PROMPT = "A person speaks calmly while facing the camera."
+NEGATIVE_PROMPT = "low quality, blurry, watermark, text"
+
+
+@pytest.fixture()
+def synthetic_audio_path(tmp_path):
+    sample_rate = 16000
+    duration_s = 2.0
+    t = np.arange(int(sample_rate * duration_s), dtype=np.float32) / sample_rate
+    audio = (0.08 * np.sin(2 * np.pi * 220 * t)).astype(np.float32)
+    path = tmp_path / "avatar_reference.wav"
+    sf.write(path, audio, sample_rate)
+    return str(path)
+
+
+@pytest.fixture()
+def synthetic_multi_audio_paths(tmp_path):
+    sample_rate = 16000
+    duration_s = 2.0
+    t = np.arange(int(sample_rate * duration_s), dtype=np.float32) / sample_rate
+    left = (0.08 * np.sin(2 * np.pi * 220 * t)).astype(np.float32)
+    right = (0.08 * np.sin(2 * np.pi * 330 * t)).astype(np.float32)
+    left_path = tmp_path / "avatar_left.wav"
+    right_path = tmp_path / "avatar_right.wav"
+    sf.write(left_path, left, sample_rate)
+    sf.write(right_path, right, sample_rate)
+    return {"person1": str(left_path), "person2": str(right_path)}
+
+
+@pytest.fixture()
+def synthetic_image_path(tmp_path):
+    path = tmp_path / "avatar_reference.png"
+    Image.new("RGB", (768, 512), (130, 104, 88)).save(path)
+    return str(path)
+
+
+def _sampling(
+    stage: str,
+    *,
+    num_frames: int = 5,
+    extra_args: dict | None = None,
+) -> OmniDiffusionSamplingParams:
+    sampling_extra_args = {
+        "stage": stage,
+        "resolution": "480p",
+        "save_fps": 25,
+        "use_distill": True,
+        "use_int8": True,
+    }
+    if extra_args:
+        sampling_extra_args.update(extra_args)
+    return OmniDiffusionSamplingParams(
+        num_frames=num_frames,
+        fps=25,
+        num_inference_steps=8,
+        guidance_scale=1.0,
+        guidance_scale_2=1.0,
+        seed=42,
+        extra_args=sampling_extra_args,
+    )
+
+
+@pytest.fixture(scope="module")
+def longcat_omni_runner() -> Generator[OmniRunner, None, None]:
+    model = prepare_longcat_video_avatar_model_for_omni(MODEL, use_int8=True)
+    with OmniRunner(
+        model,
+        seed=42,
+        stage_configs_path=None,
+        model_class_name="LongCatVideoAvatarPipeline",
+        additional_config={
+            "model_type": "avatar-v1.5",
+            "resolution": "480p",
+            "use_distill": True,
+            "use_int8": True,
+        },
+    ) as runner:
+        yield runner
+
+
+def _generate_avatar_frames(
+    omni_runner: OmniRunner,
+    *,
+    stage: str,
+    audio_path: str | dict[str, str],
+    image_path: str | None = None,
+    additional_information: dict | None = None,
+    num_frames: int = 5,
+    sampling_extra_args: dict | None = None,
+):
+    prompt = {
+        "prompt": PROMPT,
+        "negative_prompt": NEGATIVE_PROMPT,
+        "multi_modal_data": {"audio": audio_path},
+        "additional_information": additional_information or {},
+    }
+    prompt["additional_information"].update({"stage": stage, "resolution": "480p"})
+    if image_path is not None:
+        prompt["multi_modal_data"]["image"] = image_path
+
+    outputs = omni_runner.generate(
+        [prompt],
+        [_sampling(stage, num_frames=num_frames, extra_args=sampling_extra_args)],
+    )
+    assert outputs
+    output = outputs[0]
+    assert output.images
+    frames = output.images[0]
+    if isinstance(frames, tuple) and len(frames) == 2:
+        frames = frames[0]
+    if isinstance(frames, dict):
+        frames = frames.get("frames") or frames.get("video")
+    assert isinstance(frames, list)
+    return frames
+
+
+def _assert_frames(frames, *, expected_size: tuple[int, int], expected_count: int = 5):
+    assert len(frames) == expected_count
+    for frame in frames:
+        assert isinstance(frame, Image.Image)
+        assert frame.mode == "RGB"
+        assert frame.size == expected_size
+
+
+@pytest.mark.advanced_model
+@pytest.mark.diffusion
+@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": 1})
+def test_longcat_video_avatar_at2v(longcat_omni_runner: OmniRunner, synthetic_audio_path):
+    frames = _generate_avatar_frames(
+        longcat_omni_runner,
+        stage="at2v",
+        audio_path=synthetic_audio_path,
+    )
+    _assert_frames(frames, expected_size=(832, 480))
+
+
+@pytest.mark.advanced_model
+@pytest.mark.diffusion
+@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": 1})
+def test_longcat_video_avatar_ai2v(
+    longcat_omni_runner: OmniRunner,
+    synthetic_audio_path,
+    synthetic_image_path,
+):
+    frames = _generate_avatar_frames(
+        longcat_omni_runner,
+        stage="ai2v",
+        audio_path=synthetic_audio_path,
+        image_path=synthetic_image_path,
+    )
+    _assert_frames(frames, expected_size=(768, 512))
+
+
+@pytest.mark.advanced_model
+@pytest.mark.diffusion
+@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": 1})
+def test_longcat_video_avatar_multi_speaker_ai2v(
+    longcat_omni_runner: OmniRunner,
+    synthetic_multi_audio_paths,
+    synthetic_image_path,
+):
+    frames = _generate_avatar_frames(
+        longcat_omni_runner,
+        stage="ai2v",
+        audio_path=synthetic_multi_audio_paths,
+        image_path=synthetic_image_path,
+        additional_information={
+            "audio_type": "para",
+            "bbox": {
+                "person1": [32, 48, 384, 336],
+                "person2": [32, 432, 384, 720],
+            },
+        },
+    )
+    _assert_frames(frames, expected_size=(768, 512))
+
+
+@pytest.mark.advanced_model
+@pytest.mark.diffusion
+@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": 1})
+def test_longcat_video_avatar_single_speaker_ai2v_avc_continuation(
+    longcat_omni_runner: OmniRunner,
+    synthetic_audio_path,
+    synthetic_image_path,
+):
+    frames = _generate_avatar_frames(
+        longcat_omni_runner,
+        stage="ai2v",
+        audio_path=synthetic_audio_path,
+        image_path=synthetic_image_path,
+        num_frames=17,
+        sampling_extra_args={
+            "num_segments": 2,
+            "num_cond_frames": 5,
+            "use_kv_cache": True,
+        },
+    )
+    _assert_frames(frames, expected_size=(768, 512), expected_count=29)
+
+
+@pytest.mark.advanced_model
+@pytest.mark.diffusion
+@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": 1})
+def test_longcat_video_avatar_multi_speaker_ai2v_avc_continuation(
+    longcat_omni_runner: OmniRunner,
+    synthetic_multi_audio_paths,
+    synthetic_image_path,
+):
+    frames = _generate_avatar_frames(
+        longcat_omni_runner,
+        stage="ai2v",
+        audio_path=synthetic_multi_audio_paths,
+        image_path=synthetic_image_path,
+        additional_information={
+            "audio_type": "para",
+            "bbox": {
+                "person1": [32, 48, 384, 336],
+                "person2": [32, 432, 384, 720],
+            },
+        },
+        num_frames=17,
+        sampling_extra_args={
+            "num_segments": 2,
+            "num_cond_frames": 5,
+            "use_kv_cache": True,
+        },
+    )
+    _assert_frames(frames, expected_size=(768, 512), expected_count=29)

tests/model_tests/diffusion/test_alignment.py

@@ -40,6 +40,7 @@ EXCLUDED_MODELS = [
     "WanT2VDMD2Pipeline",
     "WanI2VDMD2Pipeline",
     "LongCatImagePipeline",
+    "LongCatVideoAvatarPipeline",
     "BagelPipeline",
     "BooguImagePipeline",
     "LancePipeline",

vllm_omni/diffusion/models/longcat_video/__init__.py

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from vllm_omni.diffusion.models.longcat_video.longcat_video_avatar_transformer import (
+    LongCatVideoAvatarTransformer3DModel,
+)
+from vllm_omni.diffusion.models.longcat_video.pipeline_longcat_video_avatar import (
+    LongCatVideoAvatarPipeline,
+    get_longcat_video_avatar_post_process_func,
+    get_longcat_video_avatar_pre_process_func,
+)
+
+__all__ = [
+    "LongCatVideoAvatarPipeline",
+    "LongCatVideoAvatarTransformer3DModel",
+    "get_longcat_video_avatar_post_process_func",
+    "get_longcat_video_avatar_pre_process_func",
+]

vllm_omni/diffusion/registry.py

@@ -126,6 +126,11 @@ _DIFFUSION_MODELS = {
         "pipeline_longcat_image",
         "LongCatImagePipeline",
     ),
+    "LongCatVideoAvatarPipeline": (
+        "longcat_video",
+        "pipeline_longcat_video_avatar",
+        "LongCatVideoAvatarPipeline",
+    ),
     "BagelPipeline": (
         "bagel",
         "pipeline_bagel",
@@ -525,6 +530,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = {
     "WanT2VDMD2Pipeline": "get_wan22_post_process_func",
     "WanI2VDMD2Pipeline": "get_wan22_i2v_post_process_func",
     "LongCatImagePipeline": "get_longcat_image_post_process_func",
+    "LongCatVideoAvatarPipeline": "get_longcat_video_avatar_post_process_func",
     "BagelPipeline": "get_bagel_post_process_func",
     "LancePipeline": "get_lance_post_process_func",
     "MingImagePipeline": "get_ming_image_post_process_func",
@@ -575,6 +581,7 @@ _DIFFUSION_PRE_PROCESS_FUNCS = {
     "QwenImageEditPipeline": "get_qwen_image_edit_pre_process_func",
     "QwenImageEditPlusPipeline": "get_qwen_image_edit_plus_pre_process_func",
     "LongCatImageEditPipeline": "get_longcat_image_edit_pre_process_func",
+    "LongCatVideoAvatarPipeline": "get_longcat_video_avatar_pre_process_func",
     "QwenImageLayeredPipeline": "get_qwen_image_layered_pre_process_func",
     "WanPipeline": "get_wan22_pre_process_func",
     "WanVACEPipeline": "get_wan22_vace_pre_process_func",