anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI]Remove DreamID-Omni and MagiHuman support (#6357)

Browse Source 
Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: [Your Name] <your.email@example.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wangyuandCursor authored and GitHub committed 2026-08-26 01:31:12 +00:00
1 parent 40dcfdbb43
commit aeb348d8f8
31 files changed
+26 -6957
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

examples/offline_inference/x_to_video_audio/download_dreamid_omni.py
-108
View File
@@ -1,108 +0,0 @@
import argparse
import fcntl
import os
import site
import subprocess
import tempfile
import time
from pathlib import Path
from huggingface_hub import snapshot_download
DEPENDENCY_REPO = "https://github.com/bytedance/DreamID-V.git"
DEPENDENCY_BRANCH = "omni"
CACHE_DIR = Path(tempfile.gettempdir()) / "vllm-omni-dependency"
LOCK_FILE = CACHE_DIR / ".install.lock"
DEPENDENCY_DIR = CACHE_DIR / "DreamID-Omni"
def download_dependency():
CACHE_DIR.mkdir(parents=True, exist_ok=True)
with open(LOCK_FILE, "w") as f:
fcntl.flock(f, fcntl.LOCK_EX)
if not DEPENDENCY_DIR.exists():
print(f"Downloading DreamID-Omni to {DEPENDENCY_DIR} ...")
subprocess.run(
["git", "clone", "--depth", "1", DEPENDENCY_REPO, "--branch", DEPENDENCY_BRANCH, str(DEPENDENCY_DIR)],
check=True,
)
print("Download finished.")
fcntl.flock(f, fcntl.LOCK_UN)
# write .pth to site-packages
site_packages = Path(site.getsitepackages()[0])
pth_file = site_packages / "vllm_omni_dependency.pth"
pth_file.write_text(str(DEPENDENCY_DIR))
print(f"Added {DEPENDENCY_DIR} to site-packages via {pth_file}")
def timed_download(repo_id: str, local_dir: str, allow_patterns: list | None = None):
"""Download files from HF repo and log time + destination."""
if os.path.exists(local_dir):
print(f"Directory {local_dir} already exists. Skipping download.")
return
print(f"Starting download from {repo_id} into {local_dir}")
start_time = time.time()
snapshot_download(
repo_id=repo_id,
local_dir=local_dir,
local_dir_use_symlinks=False,
allow_patterns=allow_patterns,
)
elapsed = time.time() - start_time
print(f"✅ Finished downloading {repo_id} in {elapsed:.2f} seconds. Files saved at: {local_dir}")
def main(output_dir: str):
# Wan2.2
wan_dir = os.path.join(output_dir, "Wan2.2-TI2V-5B")
timed_download(
repo_id="Wan-AI/Wan2.2-TI2V-5B",
local_dir=wan_dir,
allow_patterns=["google/*", "models_t5_umt5-xxl-enc-bf16.pth", "Wan2.2_VAE.pth"],
)
# MMAudio
mm_audio_dir = os.path.join(output_dir, "MMAudio")
timed_download(
repo_id="hkchengrex/MMAudio",
local_dir=mm_audio_dir,
allow_patterns=["ext_weights/best_netG.pt", "ext_weights/v1-16.pth"],
)
dreamid_dir = os.path.join(output_dir, "DreamID-Omni")
timed_download(repo_id="XuGuo699/DreamID-Omni", local_dir=dreamid_dir)
# Now we construct the config file
import json
data = {
"_class_name": "DreamIDOmniPipeline",
}
with open(os.path.join(output_dir, "model_index.json"), "w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
print(f"model_index.json created at {os.path.join(output_dir, 'model_index.json')}")
transformer_dir = os.path.join(output_dir, "transformer")
os.makedirs(transformer_dir, exist_ok=True)
with open(os.path.join(transformer_dir, "config.json"), "w", encoding="utf-8") as f:
json.dump({"fusion": "DreamID-Omni/dreamid_omni.safetensors"}, f)
print(f"transformer/config.json created at {os.path.join(transformer_dir, 'config.json')}")
# now we download the dependency code
download_dependency()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Download models from Hugging Face")
parser.add_argument(
"--output-dir", type=str, default="./dreamid_omni", help="Base directory to save downloaded models"
)
args = parser.parse_args()
main(args.output_dir)
examples/offline_inference/x_to_video_audio/x_to_video_audio.md
-163
View File
@@ -1,163 +0,0 @@
# X-To-Video-Audio
The `DreamID-Omni` pipeline generates short videos from text, image and video.
## Local CLI Usage
### Download the Model locally
Since DreamID-Omni combine multiple models, and without any config, so we need to download them locally.
```bash
python download_dreamid_omni.py --output-dir ./dreamid_omni
```
After download, the model directory will look like this:
```
dreamid_omni/
├── DreamID-Omni/
│ ├── dreamid_omni.safetensors
├── MMAudio/
│ ├── ext_weights/
│ │ ├── best_netG.pt
│ │ ├── v1-16.pth
├── Wan2.2-TI2V-5B/
│ ├── google/*
│ ├── models_t5_umt5-xxl-enc-bf16.pth
│ ├── Wan2.2_VAE.pth
│
├── model_index.json
└── transformer/
└── config.json # create by download_dreamid_omni.py
```
### Run the Inference
```python
python x_to_video_audio.py \
--model /path/to/dreamid_omni \
--prompt "Two people walking together and singing happily" \
--image-path ./example0.png ./example1.png \
--audio-path ./example0.wav ./example1.wav \
--video-negative-prompt "jitter, bad hands, blur, distortion" \
--audio-negative-prompt "robotic, muffled, echo, distorted" \
--cfg-parallel-size 2 \
--num-inference-steps 45 \
--height 704 \
--width 1280 \
--output out_dreamid_omni_twoip.mp4
```
In the current test scenario (2 images + 2 audio inputs), the VRAM requirement is 72GB, regardless of whether cfg-parallel is enabled or disabled.
The VRAM usage can be reduced by enabling CPU offload via --enable-cpu-offload.
For multi-GPU memory reduction on the fused DreamID-Omni transformer, you can also enable HSDP:
```python
python x_to_video_audio.py \
--model /path/to/dreamid_omni \
--prompt "Two people walking together and singing happily" \
--image-path ./example0.png ./example1.png \
--audio-path ./example0.wav ./example1.wav \
--video-negative-prompt "jitter, bad hands, blur, distortion" \
--audio-negative-prompt "robotic, muffled, echo, distorted" \
--cfg-parallel-size 2 \
--num-inference-steps 45 \
--height 704 \
--width 1280 \
--use-hsdp \
--hsdp-shard-size 2 \
--output out_dreamid_omni_twoip.mp4
```
You could take reference images/audios from the test cases in the official repo: https://github.com/Guoxu1233/DreamID-Omni
For example, single IP ref resources can be found under https://github.com/Guoxu1233/DreamID-Omni/tree/main/test_case/oneip, you could download them correspondingly to your local and use them for testing.
```python
# Example usage for oneip, ref media from the official repo DreamID-Omni
python x_to_video_audio.py \
--model /path/to/dreamid_omni \
--prompt "<img1>: In the frame, a woman with black long hair is identified as <sub1>.\n**Overall Environment/Scene**: A lively open-kitchen café at night; stove flames flare, steam rises, and warm pendant lights swing slightly as staff move behind her. The shot is an upper-body close-up.\n**Main Characters/Subjects Appearance**: <sub1> is a young woman with thick dark wavy hair and a side part. She wears a fitted black top under a light apron, a thin gold chain necklace, and small stud earrings.\n**Main Characters/Subjects Actions**: <sub1> tastes the sauce with a spoon, then turns her face toward the camera while still holding the spoon, her expression shifting from focused to conflicted.\n<sub1> maintains eye contact, swallows as if choosing her words, and says, <S>I keep telling myself I’m fine,but some nights it feels like I’m just performing calm.<E>" \
--image-path 9.png \
--audio-path 9.wav \
--video-negative-prompt "jitter, bad hands, blur, distortion" \
--audio-negative-prompt "robotic, muffled, echo, distorted" \
--cfg-parallel-size 2 \
--num-inference-steps 45 \
--height 704 \
--width 1280 \
--output out_dreamid_omni_oneip.mp4
```
## MagiHuman (`--model-type magi-human`)
MagiHuman is a text → video+audio model with a DiT MoE backbone and a ~9B-param
T5Gemma text encoder. A detailed text prompt is the only required input; an optional
image and/or audio file may be supplied for conditioning. Natively supports Tensor
Parallelism. For an 80GB node, `--tensor-parallel-size 4` is recommended to shard
the MoE weights and the text encoder.
> Install [MagiCompiler](https://github.com/SandAI-org/MagiCompiler) for correct
> attention-kernel behaviour (the pipeline otherwise falls back to stubs).
Text-only generation:
```bash
python x_to_video_audio.py \
--model-type magi-human \
--model /path/to/daVinci-MagiHuman \
--prompt "A young woman with long, wavy golden blonde hair... <dialogue and background sound>" \
--tensor-parallel-size 4 \
--height 256 --width 448 \
--num-inference-steps 8 \
--seed 52 \
--extra-body '{"seconds": 5, "sr_height": 1080, "sr_width": 1920, "sr_num_inference_steps": 5}' \
--output output_magihuman.mp4
```
With optional image and audio conditioning:
```bash
python x_to_video_audio.py \
--model-type magi-human \
--model /path/to/daVinci-MagiHuman \
--prompt "A young woman..." \
--tensor-parallel-size 4 \
--height 256 --width 448 \
--num-inference-steps 8 \
--seed 52 \
--extra-body '{"image_path": "/path/to/ref.jpg", "audio_path": "/path/to/ref.wav", "sr_height": 1080, "sr_width": 1920, "sr_num_inference_steps": 5}' \
--output output_magihuman.mp4
```
MagiHuman-specific arguments are passed as a JSON dict via `--extra-body` (declared
in `vllm_omni/model_extras/magi_human.py`, routed via `extra_args`):
- `seconds`: output duration in seconds (default 10; ignored when `audio_path` is set,
because the audio length then determines the number of frames).
- `audio_path`: path to an audio file for audio-to-video conditioning. When provided,
the audio drives the frame count and is encoded as a latent condition into the DiT
(`is_a2v` mode). Omit for pure text-to-video+audio generation.
- `image_path`: path to an image file for visual conditioning. Applied at both the
base-resolution (BR) and super-resolution (SR) stages when SR is enabled.
- `sr_height` / `sr_width`: super-resolution output resolution. SR stage is skipped
when these are omitted.
- `sr_num_inference_steps`: denoising steps for the SR stage.
## DreamID-Omni arguments
Key arguments:
- `--prompt`: text description (string).
- `--model`: path to the model local directory.
- `--height/--width`: output resolution (defaults 704 * 1024).
- `--image-path`: path to the input image list.
- `--audio-path`: path to the input audio list, indicate the timbre of the output video.
- `--cfg-parallel-size`: number of parallel cfg parallel (defaults 1).
- `--use-hsdp`: enable HSDP weight sharding for DreamID-Omni fused blocks.
- `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
- `--hsdp-replicate-size`: number of HSDP replica groups.
- `--num-inference-steps`: number of denoising steps (defaults 45).
- `--video-negative-prompt`: negative prompt for video generation.
- `--audio-negative-prompt`: negative prompt for audio generation.
- `--enable-cpu-offload`: enable CPU offload (defaults False).
- `--cache-backend`: enable `cache_dit` for acceleration.
- `--quantization`: online (dynamic) quantization method — `fp8` or `int8`.
(VAEs, the T5 text encoder, norms and modulation stay bf16.)
examples/offline_inference/x_to_video_audio/x_to_video_audio.py
-279
View File
@@ -1,279 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import json
import re
import time
from typing import Any
import numpy as np
from PIL import Image
from vllm.multimodal.media.audio import load_audio
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".gif"}
AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".m4a", ".aac", ".ogg"}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Offline inference for X -> video+audio models.")
parser.add_argument("--model", required=True, help="Model ckpt root directory.")
parser.add_argument(
"--model-type",
default="dreamid-omni",
choices=["dreamid-omni", "magi-human"],
help="Model type.",
)
parser.add_argument("--prompt", default=None, help="Text prompt.")
parser.add_argument("--image-path", type=str, nargs="+", help="list of image-path")
parser.add_argument("--audio-path", type=str, nargs="+", help="list of audio-path")
parser.add_argument("--prompt-file", type=str, default=None, help="Text prompt in json format.")
parser.add_argument(
"--extra-body",
type=str,
default=None,
help="[magi-human] JSON dict of model-specific extra params (declared in vllm_omni/model_extras/), "
'merged into sampling extra_args. Example: \'{"image_path": "/path/to/img.jpg", "seconds": 5}\'.',
)
parser.add_argument("--height", type=int, default=704, help="Video height.")
parser.add_argument("--width", type=int, default=1280, help="Video width.")
parser.add_argument("--num-inference-steps", type=int, default=45, help="Sampling steps.")
parser.add_argument("--solver-name", default="unipc", help="Solver name: unipc|dpm++|euler.")
parser.add_argument("--shift", type=float, default=5.0, help="Scheduler shift.")
parser.add_argument("--seed", type=int, default=103, help="Random seed for reproducible generation.")
parser.add_argument(
"--cfg-parallel-size",
type=int,
default=1,
choices=[1, 2, 3, 4],
help="Number of GPUs used for classifier free guidance parallel size (max 4 branches).",
)
parser.add_argument(
"--tensor-parallel-size",
type=int,
default=1,
help=("Number of GPUs used for tensor parallelism (TP) inside the DiT."),
)
parser.add_argument(
"--video-negative-prompt",
default="jitter, bad hands, blur, distortion",
help="Negative prompt for video.",
)
parser.add_argument(
"--audio-negative-prompt",
default="robotic, muffled, echo, distorted",
help="Negative prompt for audio.",
)
parser.add_argument("--output", default="dreamid_output.mp4", help="Output video path.")
parser.add_argument(
"--quantization",
type=str,
default=None,
choices=["fp8", "int8"],
help=("Online (dynamic) quantization method for the DreamID-Omni transformer. "),
)
parser.add_argument(
"--enable-cpu-offload",
action="store_true",
default=False,
help="Enable CPU offloading for diffusion models.",
)
parser.add_argument(
"--enable-layerwise-offload",
action="store_true",
help="Enable layerwise (blockwise) offloading on DiT modules.",
)
parser.add_argument("--cache-backend", type=str, default=None, choices=["cache_dit"], help="Cache backend.")
parser.add_argument(
"--use-hsdp",
action="store_true",
help="Enable HSDP for DreamID-Omni fused transformer blocks.",
)
parser.add_argument(
"--hsdp-shard-size",
type=int,
default=1,
help="Number of GPUs used for HSDP sharding when HSDP is enabled.",
)
parser.add_argument(
"--hsdp-replicate-size",
type=int,
default=1,
help="Number of HSDP replica groups. Default 1 means pure sharding.",
)
return parser.parse_args()
def load_image_and_audio(image_paths, audio_paths):
image = []
audio = []
for path in image_paths:
with Image.open(path) as img:
img = img.convert("RGB")
image.append(img)
for path in audio_paths:
audio_array, sr = load_audio(path, sr=16000)
audio_array = audio_array[int(sr * 1) : int(sr * 3)]
audio.append(audio_array)
return image, audio
def _extract_peak_memory_mb(result: Any) -> float:
"""Pull worker-reported peak VRAM (MiB) generation result.
Mirrors vllm_omni/entrypoints/openai/serving_video.py:_extract_peak_memory_mb.
"""
if isinstance(result, list):
result = result[0] if result else None
if result is None:
return 0.0
val = getattr(result, "peak_memory_mb", 0.0)
if not val:
inner = result
if isinstance(inner, list):
inner = inner[0] if inner else None
val = getattr(inner, "peak_memory_mb", 0.0)
try:
return float(val or 0.0)
except (TypeError, ValueError):
return 0.0
def main() -> None:
args = parse_args()
if args.prompt is None and args.prompt_file is None:
raise ValueError("Either --prompt or --prompt-file must be provided.")
text_prompt = args.prompt
if args.prompt_file:
with open(args.prompt_file) as f:
text_prompt = json.load(f)
text_prompt = re.sub(
r"\[SPEAKER_TIMESTAMPS_START\].*?\[SPEAKER_TIMESTAMPS_END\]", "", text_prompt, flags=re.DOTALL
).strip()
text_prompt = re.sub(
r"\[AUDIO_DESCRIPTION_START].*?\[AUDIO_DESCRIPTION_END]", "", text_prompt, flags=re.DOTALL
).strip()
text_prompt = re.sub(r"\[[A-Z_]+\]", "", text_prompt)
text_prompt = re.sub(r"\n\s*\n", "\n", text_prompt).strip()
if args.model_type == "magi-human":
prompt = text_prompt
extra_args = json.loads(args.extra_body) if args.extra_body else {}
sampling_params = OmniDiffusionSamplingParams(
height=args.height,
width=args.width,
num_inference_steps=args.num_inference_steps,
seed=args.seed,
extra_args=extra_args,
)
else:
image, audio = load_image_and_audio(args.image_path, args.audio_path)
prompt = {
"prompt": text_prompt,
"video_negative_prompt": args.video_negative_prompt,
"audio_negative_prompt": args.audio_negative_prompt,
"multi_modal_data": {"image": image, "audio": audio},
}
sampling_params = OmniDiffusionSamplingParams(
height=args.height,
width=args.width,
num_inference_steps=args.num_inference_steps,
seed=args.seed,
extra_args={
"solver_name": args.solver_name,
"shift": args.shift,
},
)
cache_config = None
if args.cache_backend == "cache_dit":
cache_config = {
"Fn_compute_blocks": 1,
"Bn_compute_blocks": 0,
"max_warmup_steps": 4,
"max_cached_steps": 20,
"residual_diff_threshold": 0.24,
"max_continuous_cached_steps": 3,
"enable_taylorseer": False,
"taylorseer_order": 1,
"scm_steps_mask_policy": None,
"scm_steps_policy": "dynamic",
}
omni_kwargs = dict(
model=args.model,
cfg_parallel_size=args.cfg_parallel_size,
tensor_parallel_size=args.tensor_parallel_size,
use_hsdp=args.use_hsdp,
hsdp_shard_size=args.hsdp_shard_size,
hsdp_replicate_size=args.hsdp_replicate_size,
model_type=args.model_type,
enable_cpu_offload=args.enable_cpu_offload,
enable_layerwise_offload=args.enable_layerwise_offload,
cache_backend=args.cache_backend,
cache_config=cache_config,
)
if args.quantization is not None:
omni_kwargs["quantization"] = args.quantization
omni = Omni(**omni_kwargs)
start = time.perf_counter()
outputs = omni.generate(prompt, sampling_params)
elapsed = time.perf_counter() - start
if not outputs:
raise RuntimeError("No output returned from the model.")
result = outputs[0]
if not result.images:
raise RuntimeError("No video frames found in model output.")
generated_video = result.images[0]
mm = result.multimodal_output or {}
generated_audio = mm.get("audio")
if args.model_type == "magi-human":
# MagiHuman returns frames already as (F, H, W, C) uint8, ready to mux.
fps = float(mm.get("fps", 25))
sample_rate = int(mm.get("audio_sample_rate", 24000))
frames = generated_video
else:
fps = float(mm.get("fps", 24))
sample_rate = int(mm.get("audio_sample_rate", 16000))
# DreamID-Omni returns video as (C, F, H, W) float32 in [-1, 1].
# mux_video_audio_bytes expects (F, H, W, C) uint8.
if not isinstance(generated_video, np.ndarray) or generated_video.ndim != 4:
raise RuntimeError(f"Unexpected video shape: {getattr(generated_video, 'shape', None)}")
frames = generated_video.transpose(1, 2, 3, 0)
frames = (np.clip((frames + 1.0) / 2.0, 0.0, 1.0) * 255.0).round().astype(np.uint8)
audio_np = None
if generated_audio is not None:
audio_np = np.squeeze(np.asarray(generated_audio)).astype(np.float32)
output_path = args.output
video_bytes = mux_video_audio_bytes(
frames,
audio_np,
fps=float(fps),
audio_sample_rate=sample_rate,
)
with open(output_path, "wb") as f:
f.write(video_bytes)
print(f"Saved generated video to {output_path}")
print(f"Total time: {elapsed:.2f}s")
peak_mb = _extract_peak_memory_mb(outputs)
if peak_mb:
print(f"Worker peak GPU memory (reserved): {peak_mb:.2f} MiB ({peak_mb / 1024:.2f} GiB)")
if __name__ == "__main__":
main()