[Diffusion] [Model] Dreamid omni (#1855)

Signed-off-by: dengyunyang <584797741@qq.com>

examples/offline_inference/x_to_video_audio/download_dreamid_omni.py

@@ -0,0 +1,103 @@
+import argparse
+import fcntl
+import os
+import site
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+
+from huggingface_hub import snapshot_download
+
+DEPENDENCY_REPO = "https://github.com/bytedance/DreamID-V.git"
+DEPENDENCY_BRANCH = "omni"
+CACHE_DIR = Path(tempfile.gettempdir()) / "vllm-omni-dependency"
+LOCK_FILE = CACHE_DIR / ".install.lock"
+DEPENDENCY_DIR = CACHE_DIR / "DreamID-Omni"
+
+
+def download_dependency():
+    CACHE_DIR.mkdir(parents=True, exist_ok=True)
+
+    with open(LOCK_FILE, "w") as f:
+        fcntl.flock(f, fcntl.LOCK_EX)
+        if not DEPENDENCY_DIR.exists():
+            print(f"Downloading DreamID-Omni to {DEPENDENCY_DIR} ...")
+            subprocess.run(
+                ["git", "clone", "--depth", "1", DEPENDENCY_REPO, "--branch", DEPENDENCY_BRANCH, str(DEPENDENCY_DIR)],
+                check=True,
+            )
+            print("Download finished.")
+        fcntl.flock(f, fcntl.LOCK_UN)
+
+    # write .pth to site-packages
+    site_packages = Path(site.getsitepackages()[0])
+    pth_file = site_packages / "vllm_omni_dependency.pth"
+    pth_file.write_text(str(DEPENDENCY_DIR))
+    print(f"Added {DEPENDENCY_DIR} to site-packages via {pth_file}")
+
+
+def timed_download(repo_id: str, local_dir: str, allow_patterns: list | None = None):
+    """Download files from HF repo and log time + destination."""
+    if os.path.exists(local_dir):
+        print(f"Directory {local_dir} already exists. Skipping download.")
+        return
+    print(f"Starting download from {repo_id} into {local_dir}")
+    start_time = time.time()
+
+    snapshot_download(
+        repo_id=repo_id,
+        local_dir=local_dir,
+        local_dir_use_symlinks=False,
+        allow_patterns=allow_patterns,
+    )
+
+    elapsed = time.time() - start_time
+    print(f"✅ Finished downloading {repo_id} in {elapsed:.2f} seconds. Files saved at: {local_dir}")
+
+
+def main(output_dir: str):
+    # Wan2.2
+    wan_dir = os.path.join(output_dir, "Wan2.2-TI2V-5B")
+    timed_download(
+        repo_id="Wan-AI/Wan2.2-TI2V-5B",
+        local_dir=wan_dir,
+        allow_patterns=["google/*", "models_t5_umt5-xxl-enc-bf16.pth", "Wan2.2_VAE.pth"],
+    )
+
+    # MMAudio
+    mm_audio_dir = os.path.join(output_dir, "MMAudio")
+    timed_download(
+        repo_id="hkchengrex/MMAudio",
+        local_dir=mm_audio_dir,
+        allow_patterns=["ext_weights/best_netG.pt", "ext_weights/v1-16.pth"],
+    )
+
+    dreamid_dir = os.path.join(output_dir, "DreamID-Omni")
+
+    timed_download(repo_id="XuGuo699/DreamID-Omni", local_dir=dreamid_dir)
+
+    # Now we construct the config file
+    import json
+
+    data = {
+        "_class_name": "DreamIDOmniPipeline",
+        "fusion": "DreamID-Omni/dreamid_omni.safetensors",
+    }
+
+    with open(os.path.join(output_dir, "model_index.json"), "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2)
+
+    print(f"model_index.json created at {os.path.join(output_dir, 'model_index.json')}")
+
+    # now we download the dependency code
+    download_dependency()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Download models from Hugging Face")
+    parser.add_argument(
+        "--output-dir", type=str, default="./dreamid_omni", help="Base directory to save downloaded models"
+    )
+    args = parser.parse_args()
+    main(args.output_dir)

examples/offline_inference/x_to_video_audio/x_to_video_audio.md

@@ -0,0 +1,58 @@
+# X-To-Video-Audio
+
+The `DreamID-Omni` pipeline generates short videos from text, image and video.
+
+## Local CLI Usage
+### Download the Model locally
+Since DreamID-Omni combine multiple models, and without any config, so we need to download them locally.
+
+```bash
+python download_dreamid_omni.py --output-dir ./dreamid_omni
+```
+After download, the model directory will look like this:
+
+```
+dreamid_omni/
+├── DreamID-Omni/
+│   ├── dreamid_omni.safetensors
+├── MMAudio/
+│   ├── ext_weights/
+│   │   ├── best_netG.pt
+│   │   ├── v1-16.pth
+├── Wan2.2-TI2V-5B/
+│   ├── google/*
+│   ├── models_t5_umt5-xxl-enc-bf16.pth
+│   ├── Wan2.2_VAE.pth
+│
+├── model_index.json # create by download_dreamid_omni.py
+```
+
+### Run the Inference
+```
+python x_to_video_audio.py \
+  --model /xx/dreamid_omni \
+  --prompt "Two people walking together and singing happily" \
+  --image-path ./example0.png ./example1.png \
+  --audio-path ./example0.wav ./example1.wav \
+  --video-negative-prompt "jitter, bad hands, blur, distortion" \
+  --audio-negative-prompt "robotic, muffled, echo, distorted" \
+  --cfg-parallel-size 2 \
+  --num-inference-steps 45 \
+  --height 704 \
+  --width 1280 \
+  --output dreamid_omni.mp4
+```
+In the current test scenario (2 images + 2 audio inputs), the VRAM requirement is 72GB, regardless of whether cfg-parallel is enabled or disabled.
+The VRAM usage can be reduced by enabling CPU offload via --enable-cpu-offload.
+
+Key arguments:
+- `--prompt`: text description (string).
+- `--model`: path to the model local directory.
+- `--height/--width`: output resolution (defaults 704 * 1024).
+- `--image-path`: path to the input image list.
+- `--audio-path`: path to the input audio list, indicate the timbre of the output video.
+- `--cfg-parallel-size`: number of parallel cfg parallel (defaults 1).
+- `--num-inference-steps`: number of denoising steps (defaults 45).
+- `--video-negative-prompt`: negative prompt for video generation.
+- `--audio-negative-prompt`: negative prompt for audio generation.
+- `--enable-cpu-offload`: enable CPU offload (defaults False).

examples/offline_inference/x_to_video_audio/x_to_video_audio.py

@@ -0,0 +1,148 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import argparse
+import re
+import time
+
+import librosa
+from PIL import Image
+
+from vllm_omni.diffusion.data import DiffusionParallelConfig
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".gif"}
+AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".m4a", ".aac", ".ogg"}
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Offline inference for DreamID-Omni (video + audio).")
+    parser.add_argument("--model", required=True, help="DreamID ckpt root directory.")
+    parser.add_argument("--model-type", default="dreamid-omni", help="Model type.")
+    parser.add_argument("--prompt", default=None, help="Text prompt.")
+
+    parser.add_argument("--image-path", type=str, nargs="+", help="list of image-path")
+    parser.add_argument("--audio-path", type=str, nargs="+", help="list of audio-path")
+    parser.add_argument("--prompt-file", type=str, default=None, help="Text prompt in json format.")
+
+    parser.add_argument("--height", type=int, default=704, help="Video height.")
+    parser.add_argument("--width", type=int, default=1280, help="Video width.")
+    parser.add_argument("--num-inference-steps", type=int, default=45, help="Sampling steps.")
+    parser.add_argument("--solver-name", default="unipc", help="Solver name: unipc|dpm++|euler.")
+    parser.add_argument("--shift", type=float, default=5.0, help="Scheduler shift.")
+    parser.add_argument("--seed", type=int, default=103, help="Random seed for reproducible generation.")
+    parser.add_argument(
+        "--cfg-parallel-size",
+        type=int,
+        default=1,
+        choices=[1, 2],
+        help="Number of GPUs used for classifier free guidance parallel size.",
+    )
+    parser.add_argument(
+        "--video-negative-prompt",
+        default="jitter, bad hands, blur, distortion",
+        help="Negative prompt for video.",
+    )
+    parser.add_argument(
+        "--audio-negative-prompt",
+        default="robotic, muffled, echo, distorted",
+        help="Negative prompt for audio.",
+    )
+    parser.add_argument("--output", default="dreamid_output.mp4", help="Output video path.")
+    parser.add_argument(
+        "--enable-cpu-offload",
+        action="store_true",
+        default=False,
+        help="Enable CPU offloading for diffusion models.",
+    )
+    return parser.parse_args()
+
+
+def load_image_and_audio(image_paths, audio_paths):
+    image = []
+    audio = []
+
+    for path in image_paths:
+        with Image.open(path) as img:
+            img = img.convert("RGB")
+            image.append(img)
+
+    for path in audio_paths:
+        audio_array, sr = librosa.load(path, sr=16000)
+        audio_array = audio_array[int(sr * 1) : int(sr * 3)]
+        audio.append(audio_array)
+    return image, audio
+
+
+def main() -> None:
+    args = parse_args()
+    if args.prompt is None and args.prompt_file is None:
+        raise ValueError("Either --prompt or --prompt-file must be provided.")
+
+    text_prompt = args.prompt
+    if args.prompt_file:
+        import json
+
+        with open(args.prompt_file) as f:
+            text_prompt = json.load(f)
+            text_prompt = re.sub(
+                r"\[SPEAKER_TIMESTAMPS_START\].*?\[SPEAKER_TIMESTAMPS_END\]", "", text_prompt, flags=re.DOTALL
+            ).strip()
+            text_prompt = re.sub(
+                r"\[AUDIO_DESCRIPTION_START].*?\[AUDIO_DESCRIPTION_END]", "", text_prompt, flags=re.DOTALL
+            ).strip()
+            text_prompt = re.sub(r"\[[A-Z_]+\]", "", text_prompt)
+            text_prompt = re.sub(r"\n\s*\n", "\n", text_prompt).strip()
+
+    image, audio = load_image_and_audio(args.image_path, args.audio_path)
+
+    prompt = {
+        "prompt": text_prompt,
+        "video_negative_prompt": args.video_negative_prompt,
+        "audio_negative_prompt": args.audio_negative_prompt,
+        "multi_modal_data": {"image": image, "audio": audio},
+    }
+
+    sampling_params = OmniDiffusionSamplingParams(
+        height=args.height,
+        width=args.width,
+        num_inference_steps=args.num_inference_steps,
+        seed=args.seed,
+        extra_args={
+            "solver_name": args.solver_name,
+            "shift": args.shift,
+        },
+    )
+
+    parallel_config = DiffusionParallelConfig(
+        cfg_parallel_size=args.cfg_parallel_size,
+    )
+
+    omni = Omni(
+        model=args.model,
+        parallel_config=parallel_config,
+        model_type=args.model_type,
+        enable_cpu_offload=args.enable_cpu_offload,
+    )
+    start = time.perf_counter()
+    outputs = omni.generate(prompt, sampling_params)
+    elapsed = time.perf_counter() - start
+
+    if not outputs:
+        raise RuntimeError("No output returned from DreamID-Omni.")
+    output = outputs[0].request_output
+    generated_video = output[0].images[0][0]
+    generated_audio = output[0].images[0][1]
+    try:
+        from dreamid_omni.utils.io_utils import save_video
+    except Exception as e:
+        raise RuntimeError(f"Failed to extract video and audio from DreamID-Omni output. Error: {e}")
+    output_path = args.output
+    save_video(output_path, generated_video, generated_audio, fps=24, sample_rate=16000)
+    print(f"Saved generated video to {output_path}")
+    print(f"Total time: {elapsed:.2f}s")
+
+
+if __name__ == "__main__":
+    main()