diff --git a/examples/offline_inference/magi_human/README.md b/examples/offline_inference/magi_human/README.md deleted file mode 100644 index 2b89093d9..000000000 --- a/examples/offline_inference/magi_human/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# MagiHuman Generation - -MagiHuman is an advanced, omni-modality model that generates both high-quality video and lip-synced audio from a text prompt. - -Because MagiHuman is a very large model featuring a powerful DiT MoE backbone and a ~9B parameter T5Gemma text encoder, it natively supports **Tensor Parallelism (TP)** in vLLM-Omni to run efficiently across multi-GPU setups, reducing device memory bottlenecks. - -## Setup - -### Install MagiCompiler (recommended) - -MagiHuman relies on [MagiCompiler](https://github.com/SandAI-org/MagiCompiler) for custom-op registration used by the DiT attention kernels. While the pipeline can fall back to stub implementations, installing MagiCompiler is **strongly recommended** for correct behaviour. - -```bash -# Clone the repo -git clone https://github.com/SandAI-org/MagiCompiler.git -cd MagiCompiler - -# System dependencies (optional, for FX graph visualization; Debian/Ubuntu) -sudo apt update && sudo apt install -y graphviz - -# Python dependencies -pip install -r requirements.txt - -# Install MagiCompiler -pip install . # end users (recommended) -# pip install -e . # developers (editable install) -``` - -### Hardware requirements - -Ensure your hardware has enough VRAM. For a standard node with 80GB GPUs, running with `--tensor-parallel-size 4` is recommended to shard both the MoE weights and the T5Gemma text encoder across 4 GPUs, reducing the per-GPU peak VRAM overhead significantly (by roughly ~13.5GB per GPU compared to single-device inference). - -Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for further details on allocating memory. - -## Run Examples - -Get into the example folder: -```bash -cd examples/offline_inference/magi_human -``` - -### End-to-End Generation (Text to Video+Audio) - -Generate a video with synchronized speech natively generated by the model. - -```bash -python end2end.py \ - --model /proj-tango-pvc/users/zhipeng.wang/workspace/models/daVinci-MagiHuman \ - --prompt "A young woman with long, wavy golden blonde hair..." \ - --tensor-parallel-size 4 \ - --output output_magihuman.mp4 -``` - -## Common Parameters - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `--model` | *(Required)* | Local model path or HuggingFace ID | -| `--prompt` | *(built-in demo prompt)* | Highly detailed text prompt dictating visual look and dialogue text | -| `--tensor-parallel-size` | `4` | Tensor parallelism size (Number of GPUs) | -| `--height` | `256` | Initial resolution height | -| `--width` | `448` | Initial resolution width | -| `--num-inference-steps` | `8` | Denoising steps | -| `--seed` | `52` | Random seed | -| `--output` | `output_magihuman.mp4` | Output video with audio path | - -## Example materials - -??? abstract "end2end.py" - ``````py - --8<-- "examples/offline_inference/magi_human/end2end.py" - `````` diff --git a/examples/offline_inference/magi_human/end2end.py b/examples/offline_inference/magi_human/end2end.py deleted file mode 100644 index 64f11c465..000000000 --- a/examples/offline_inference/magi_human/end2end.py +++ /dev/null @@ -1,122 +0,0 @@ -import argparse - -from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes -from vllm_omni.entrypoints.omni import Omni -from vllm_omni.inputs.data import OmniDiffusionSamplingParams - - -def parse_args(): - parser = argparse.ArgumentParser(description="End-to-end inference script for MagiHuman.") - parser.add_argument("--model", type=str, required=True, help="Path or ID of the MagiHuman model.") - parser.add_argument( - "--prompt", - type=str, - default="", - help="Text prompt containing visual description, dialogue, and background sound.", - ) - parser.add_argument( - "--tensor-parallel-size", "-tp", type=int, default=4, help="Tensor parallel size (number of GPUs)." - ) - parser.add_argument( - "--output", type=str, default="output_magihuman.mp4", help="Path to save the generated mp4 file." - ) - parser.add_argument("--height", type=int, default=256, help="Video height.") - parser.add_argument("--width", type=int, default=448, help="Video width.") - parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.") - parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.") - return parser.parse_args() - - -def main(): - args = parse_args() - - print(f"Initializing MagiHuman pipeline with TP={args.tensor_parallel_size}...") - omni = Omni( - model=args.model, - init_timeout=1200, - tensor_parallel_size=args.tensor_parallel_size, - devices=list(range(args.tensor_parallel_size)), - ) - - prompt = args.prompt - if not prompt: - prompt = ( - "A young woman with long, wavy golden blonde hair and bright blue eyes, " - "wearing a fitted ivory silk blouse with a delicate lace collar, sits " - "stationary in front of a softly lit, blurred warm-toned interior. Her " - "overall disposition is warm, composed, and gently confident. The camera " - "holds a static medium close-up, framing her from the shoulders up, " - "with shallow depth of field keeping her face in sharp focus. Soft " - "directional key light falls from the upper left, casting a gentle " - "highlight along her cheekbone and nose bridge. She draws a quiet breath, " - "the levator labii superiors relaxing as her lips part. She speaks in " - "clear, warm, unhurried American English: " - "\"The most beautiful things in life aren't things at all — " - "they're moments, feelings, and the people who make you feel truly alive.\" " - "Her jaw descends smoothly on each stressed syllable; the orbicularis oris " - "shapes each vowel with precision. A faint, genuine smile engages the " - "zygomaticus major, lifting her lip corners fractionally. Her brows rest " - "in a soft, neutral arch throughout. She maintains steady, forward-facing " - "eye contact. Head position remains level; no torso displacement occurs.\n\n" - "Dialogue:\n" - ": " - "\"The most beautiful things in life aren't things at all — " - "they're moments, feelings, and the people who make you feel truly alive.\"\n\n" - "Background Sound:\n" - "" - ) - - sampling_params = OmniDiffusionSamplingParams( - height=args.height, - width=args.width, - num_inference_steps=args.num_inference_steps, - seed=args.seed, - extra_args={ - "seconds": 5, - "sr_height": 1080, - "sr_width": 1920, - "sr_num_inference_steps": 5, - }, - ) - - print(f"Generating with prompt: {prompt[:80]}...") - outputs = omni.generate( - prompts=[prompt], - sampling_params_list=[sampling_params], - ) - - print(f"Generation complete. Output type: {type(outputs)}") - if outputs: - first = outputs[0] - - if hasattr(first, "images") and first.images: - video_frames = first.images[0] - print(f"Video frames: shape={video_frames.shape}, dtype={video_frames.dtype}") - - audio_waveform = None - mm = first.multimodal_output or {} - if mm: - audio_waveform = mm.get("audio") - if audio_waveform is not None: - print(f"Audio waveform: shape={audio_waveform.shape}, dtype={audio_waveform.dtype}") - - output_fps = float(mm.get("fps", 25)) - output_sr = int(mm.get("audio_sample_rate", 24000)) - print(f"Using fps={output_fps}, audio_sample_rate={output_sr} from model output") - - video_bytes = mux_video_audio_bytes( - video_frames, - audio_waveform, - fps=output_fps, - audio_sample_rate=output_sr, - ) - with open(args.output, "wb") as f: - f.write(video_bytes) - print(f"Saved MP4 ({len(video_bytes)} bytes) to {args.output}") - print("SUCCESS: MagiHuman pipeline generation completed.") - else: - print("WARNING: No outputs returned.") - - -if __name__ == "__main__": - main() diff --git a/examples/offline_inference/x_to_video_audio/x_to_video_audio.md b/examples/offline_inference/x_to_video_audio/x_to_video_audio.md index f5071ce6d..8305c785f 100644 --- a/examples/offline_inference/x_to_video_audio/x_to_video_audio.md +++ b/examples/offline_inference/x_to_video_audio/x_to_video_audio.md @@ -87,6 +87,63 @@ python x_to_video_audio.py \ ``` +## MagiHuman (`--model-type magi-human`) + +MagiHuman is a text → video+audio model with a DiT MoE backbone and a ~9B-param +T5Gemma text encoder. A detailed text prompt is the only required input; an optional +image and/or audio file may be supplied for conditioning. Natively supports Tensor +Parallelism. For an 80GB node, `--tensor-parallel-size 4` is recommended to shard +the MoE weights and the text encoder. + +> Install [MagiCompiler](https://github.com/SandAI-org/MagiCompiler) for correct +> attention-kernel behaviour (the pipeline otherwise falls back to stubs). + +Text-only generation: + +```bash +python x_to_video_audio.py \ + --model-type magi-human \ + --model /path/to/daVinci-MagiHuman \ + --prompt "A young woman with long, wavy golden blonde hair... " \ + --tensor-parallel-size 4 \ + --height 256 --width 448 \ + --num-inference-steps 8 \ + --seed 52 \ + --extra-body '{"seconds": 5, "sr_height": 1080, "sr_width": 1920, "sr_num_inference_steps": 5}' \ + --output output_magihuman.mp4 +``` + +With optional image and audio conditioning: + +```bash +python x_to_video_audio.py \ + --model-type magi-human \ + --model /path/to/daVinci-MagiHuman \ + --prompt "A young woman..." \ + --tensor-parallel-size 4 \ + --height 256 --width 448 \ + --num-inference-steps 8 \ + --seed 52 \ + --extra-body '{"image_path": "/path/to/ref.jpg", "audio_path": "/path/to/ref.wav", "sr_height": 1080, "sr_width": 1920, "sr_num_inference_steps": 5}' \ + --output output_magihuman.mp4 +``` + +MagiHuman-specific arguments are passed as a JSON dict via `--extra-body` (declared +in `vllm_omni/model_extras/magi_human.py`, routed via `extra_args`): + +- `seconds`: output duration in seconds (default 10; ignored when `audio_path` is set, + because the audio length then determines the number of frames). +- `audio_path`: path to an audio file for audio-to-video conditioning. When provided, + the audio drives the frame count and is encoded as a latent condition into the DiT + (`is_a2v` mode). Omit for pure text-to-video+audio generation. +- `image_path`: path to an image file for visual conditioning. Applied at both the + base-resolution (BR) and super-resolution (SR) stages when SR is enabled. +- `sr_height` / `sr_width`: super-resolution output resolution. SR stage is skipped + when these are omitted. +- `sr_num_inference_steps`: denoising steps for the SR stage. + +## DreamID-Omni arguments + Key arguments: - `--prompt`: text description (string). - `--model`: path to the model local directory. diff --git a/examples/offline_inference/x_to_video_audio/x_to_video_audio.py b/examples/offline_inference/x_to_video_audio/x_to_video_audio.py index 30de8aa45..0efbdc37d 100644 --- a/examples/offline_inference/x_to_video_audio/x_to_video_audio.py +++ b/examples/offline_inference/x_to_video_audio/x_to_video_audio.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import argparse +import json import re import time from typing import Any @@ -20,14 +21,26 @@ AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".m4a", ".aac", ".ogg"} def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Offline inference for DreamID-Omni (video + audio).") - parser.add_argument("--model", required=True, help="DreamID ckpt root directory.") - parser.add_argument("--model-type", default="dreamid-omni", help="Model type.") + parser = argparse.ArgumentParser(description="Offline inference for X -> video+audio models.") + parser.add_argument("--model", required=True, help="Model ckpt root directory.") + parser.add_argument( + "--model-type", + default="dreamid-omni", + choices=["dreamid-omni", "magi-human"], + help="Model type.", + ) parser.add_argument("--prompt", default=None, help="Text prompt.") parser.add_argument("--image-path", type=str, nargs="+", help="list of image-path") parser.add_argument("--audio-path", type=str, nargs="+", help="list of audio-path") parser.add_argument("--prompt-file", type=str, default=None, help="Text prompt in json format.") + parser.add_argument( + "--extra-body", + type=str, + default=None, + help="[magi-human] JSON dict of model-specific extra params (declared in vllm_omni/model_extras/), " + 'merged into sampling extra_args. Example: \'{"image_path": "/path/to/img.jpg", "seconds": 5}\'.', + ) parser.add_argument("--height", type=int, default=704, help="Video height.") parser.add_argument("--width", type=int, default=1280, help="Video width.") @@ -142,8 +155,6 @@ def main() -> None: text_prompt = args.prompt if args.prompt_file: - import json - with open(args.prompt_file) as f: text_prompt = json.load(f) text_prompt = re.sub( @@ -155,25 +166,36 @@ def main() -> None: text_prompt = re.sub(r"\[[A-Z_]+\]", "", text_prompt) text_prompt = re.sub(r"\n\s*\n", "\n", text_prompt).strip() - image, audio = load_image_and_audio(args.image_path, args.audio_path) + if args.model_type == "magi-human": + prompt = text_prompt + extra_args = json.loads(args.extra_body) if args.extra_body else {} + sampling_params = OmniDiffusionSamplingParams( + height=args.height, + width=args.width, + num_inference_steps=args.num_inference_steps, + seed=args.seed, + extra_args=extra_args, + ) + else: + image, audio = load_image_and_audio(args.image_path, args.audio_path) - prompt = { - "prompt": text_prompt, - "video_negative_prompt": args.video_negative_prompt, - "audio_negative_prompt": args.audio_negative_prompt, - "multi_modal_data": {"image": image, "audio": audio}, - } + prompt = { + "prompt": text_prompt, + "video_negative_prompt": args.video_negative_prompt, + "audio_negative_prompt": args.audio_negative_prompt, + "multi_modal_data": {"image": image, "audio": audio}, + } - sampling_params = OmniDiffusionSamplingParams( - height=args.height, - width=args.width, - num_inference_steps=args.num_inference_steps, - seed=args.seed, - extra_args={ - "solver_name": args.solver_name, - "shift": args.shift, - }, - ) + sampling_params = OmniDiffusionSamplingParams( + height=args.height, + width=args.width, + num_inference_steps=args.num_inference_steps, + seed=args.seed, + extra_args={ + "solver_name": args.solver_name, + "shift": args.shift, + }, + ) parallel_config = DiffusionParallelConfig( cfg_parallel_size=args.cfg_parallel_size, @@ -215,22 +237,28 @@ def main() -> None: elapsed = time.perf_counter() - start if not outputs: - raise RuntimeError("No output returned from DreamID-Omni.") + raise RuntimeError("No output returned from the model.") result = outputs[0] if not result.images: - raise RuntimeError("No video frames found in DreamID-Omni output.") + raise RuntimeError("No video frames found in model output.") generated_video = result.images[0] mm = result.multimodal_output or {} generated_audio = mm.get("audio") - fps = int(mm.get("fps", 24)) - sample_rate = int(mm.get("audio_sample_rate", 16000)) - # DreamID-Omni returns video as (C, F, H, W) float32 in [-1, 1]. - # mux_video_audio_bytes expects (F, H, W, C) uint8. - if not isinstance(generated_video, np.ndarray) or generated_video.ndim != 4: - raise RuntimeError(f"Unexpected video shape: {getattr(generated_video, 'shape', None)}") - frames = generated_video.transpose(1, 2, 3, 0) - frames = (np.clip((frames + 1.0) / 2.0, 0.0, 1.0) * 255.0).round().astype(np.uint8) + if args.model_type == "magi-human": + # MagiHuman returns frames already as (F, H, W, C) uint8, ready to mux. + fps = float(mm.get("fps", 25)) + sample_rate = int(mm.get("audio_sample_rate", 24000)) + frames = generated_video + else: + fps = float(mm.get("fps", 24)) + sample_rate = int(mm.get("audio_sample_rate", 16000)) + # DreamID-Omni returns video as (C, F, H, W) float32 in [-1, 1]. + # mux_video_audio_bytes expects (F, H, W, C) uint8. + if not isinstance(generated_video, np.ndarray) or generated_video.ndim != 4: + raise RuntimeError(f"Unexpected video shape: {getattr(generated_video, 'shape', None)}") + frames = generated_video.transpose(1, 2, 3, 0) + frames = (np.clip((frames + 1.0) / 2.0, 0.0, 1.0) * 255.0).round().astype(np.uint8) audio_np = None if generated_audio is not None: diff --git a/tests/model_extras/test_model_extras.py b/tests/model_extras/test_model_extras.py index a75dd3e44..22ce82bc2 100644 --- a/tests/model_extras/test_model_extras.py +++ b/tests/model_extras/test_model_extras.py @@ -57,6 +57,23 @@ def test_sensenova_extra_registry_declares_request_and_response_params() -> None assert should_init_extra_args_for_non_diffusion_stages("SenseNovaU1Pipeline") is False +@pytest.mark.core_model +@pytest.mark.cpu +def test_magi_human_extra_registry_declares_request_and_response_params() -> None: + assert get_extra_body_params("MagiHumanPipeline") == frozenset( + { + "seconds", + "audio_path", + "image_path", + "sr_height", + "sr_width", + "sr_num_inference_steps", + } + ) + assert get_extra_output_params("MagiHumanPipeline") == frozenset() + assert should_init_extra_args_for_non_diffusion_stages("MagiHumanPipeline") is False + + @pytest.mark.core_model @pytest.mark.cpu def test_helios_extra_registry_declares_request_and_response_params() -> None: diff --git a/vllm_omni/diffusion/models/t5_encoder/t5_gemma_encoder.py b/vllm_omni/diffusion/models/t5_encoder/t5_gemma_encoder.py index eca4267fa..f7f62f8ce 100644 --- a/vllm_omni/diffusion/models/t5_encoder/t5_gemma_encoder.py +++ b/vllm_omni/diffusion/models/t5_encoder/t5_gemma_encoder.py @@ -157,7 +157,7 @@ class T5GemmaEncoderLayer(nn.Module): num_kv_heads=config.num_key_value_heads, head_dim=config.head_dim, max_position_embeddings=config.max_position_embeddings, - rope_theta=config.rope_theta, + rope_theta=getattr(config, "rope_theta", 10000.0), ) self.mlp = T5GemmaMLP( hidden_size=config.hidden_size, diff --git a/vllm_omni/model_extras/magi_human.py b/vllm_omni/model_extras/magi_human.py new file mode 100644 index 000000000..470a1455b --- /dev/null +++ b/vllm_omni/model_extras/magi_human.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +# MagiHuman is a text -> video+audio model. Its model-specific knobs are read +# from sampling_params.extra_args in pipeline_magi_human.py; declaring them here +# routes request `extra_body` fields into OmniDiffusionSamplingParams.extra_args +# so the model can be driven through the standard task example. +MAGI_HUMAN_EXTRA_BODY_PARAMS = frozenset( + { + "seconds", + "audio_path", + "image_path", + "sr_height", + "sr_width", + "sr_num_inference_steps", + } +) +MAGI_HUMAN_EXTRA_OUTPUT_PARAMS = frozenset() diff --git a/vllm_omni/model_extras/registry.py b/vllm_omni/model_extras/registry.py index 7add6eb0d..5c5f17b45 100644 --- a/vllm_omni/model_extras/registry.py +++ b/vllm_omni/model_extras/registry.py @@ -23,6 +23,10 @@ from vllm_omni.model_extras.helios import ( HELIOS_EXTRA_BODY_PARAMS, HELIOS_EXTRA_OUTPUT_PARAMS, ) +from vllm_omni.model_extras.magi_human import ( + MAGI_HUMAN_EXTRA_BODY_PARAMS, + MAGI_HUMAN_EXTRA_OUTPUT_PARAMS, +) from vllm_omni.model_extras.sensenova_u1 import ( SENSENOVA_U1_EXTRA_BODY_PARAMS, SENSENOVA_U1_EXTRA_OUTPUT_PARAMS, @@ -80,6 +84,10 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { "extra_body_params": SENSENOVA_U1_EXTRA_BODY_PARAMS, "extra_output_params": SENSENOVA_U1_EXTRA_OUTPUT_PARAMS, }, + "MagiHumanPipeline": { + "extra_body_params": MAGI_HUMAN_EXTRA_BODY_PARAMS, + "extra_output_params": MAGI_HUMAN_EXTRA_OUTPUT_PARAMS, + }, "HeliosPipeline": { "extra_body_params": HELIOS_EXTRA_BODY_PARAMS, "extra_output_params": HELIOS_EXTRA_OUTPUT_PARAMS,