Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Hongsheng Liu
|
d32625de81 | ||
|
Hongsheng Liu
|
8c994e57fd | ||
|
Hongsheng Liu
|
13731101ab | ||
|
R0CKSTAR
|
636d44e272 | ||
|
yiminghub2024
|
8e44e78038 | ||
|
Hongsheng Liu
|
b98b7c8529 |
No files matched your search
+4
-14
@@ -16,31 +16,21 @@ nav:
|
||||
- Examples:
|
||||
- examples/README.md
|
||||
- Offline Inference:
|
||||
- GLM-Image Multistage End-to-End Inference: user_guide/examples/offline_inference/glm_image.md
|
||||
- HunyuanImage-3.0 Image-to-Text Inference: user_guide/examples/offline_inference/hunyuan_image3.md
|
||||
- Image-To-Image: user_guide/examples/offline_inference/image_to_image.md
|
||||
- Image-To-Video: user_guide/examples/offline_inference/image_to_video.md
|
||||
- InternVLA-A1: user_guide/examples/offline_inference/internvla_a1.md
|
||||
- MiMo-Audio: user_guide/examples/offline_inference/mimo_audio.md
|
||||
- Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
|
||||
- Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md
|
||||
- Speech-To-Video: user_guide/examples/offline_inference/speech_to_video.md
|
||||
- Text-To-Audio: user_guide/examples/offline_inference/text_to_audio.md
|
||||
- Text-To-Image: user_guide/examples/offline_inference/text_to_image.md
|
||||
- Text-To-Speech: user_guide/examples/offline_inference/text_to_speech.md
|
||||
- Text-To-Video: user_guide/examples/offline_inference/text_to_video.md
|
||||
- X-To-Text: user_guide/examples/offline_inference/x_to_text.md
|
||||
- X-To-Video-Audio: user_guide/examples/offline_inference/x_to_video_audio.md
|
||||
- Online Serving:
|
||||
- BAGEL-7B-MoT: user_guide/examples/online_serving/bagel.md
|
||||
- vLLM-Omni Helm Chart: user_guide/examples/online_serving/chart-helm.md
|
||||
- Diffusers Backend Adapter: user_guide/examples/online_serving/diffusers_pipeline_adapter.md
|
||||
- GLM-Image Online Serving: user_guide/examples/online_serving/glm_image.md
|
||||
- Image-To-Image: user_guide/examples/online_serving/image_to_image.md
|
||||
- Image-To-Video: user_guide/examples/online_serving/image_to_video.md
|
||||
- MiMo-Audio: user_guide/examples/online_serving/mimo_audio.md
|
||||
- Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md
|
||||
- Qwen3-Omni: user_guide/examples/online_serving/qwen3_omni.md
|
||||
- Text-To-Speech: user_guide/examples/online_serving/text_to_speech.md
|
||||
- Speech-To-Video: user_guide/examples/online_serving/speech_to_video.md
|
||||
- Text-To-Image: user_guide/examples/online_serving/text_to_image.md
|
||||
- Text-To-Speech: user_guide/examples/online_serving/text_to_speech.md
|
||||
- Text-To-Video: user_guide/examples/online_serving/text_to_video.md
|
||||
- General:
|
||||
- usage/*
|
||||
|
||||
@@ -24,6 +24,23 @@ SERVING_MODE_TITLES = {
|
||||
"online_serving": "Online serving",
|
||||
}
|
||||
|
||||
# Keep the Examples navigation focused on reusable modality/task entry points.
|
||||
# Model-specific scripts remain available under ``examples/`` and are surfaced
|
||||
# from these shared pages instead of getting one page per model.
|
||||
GENERAL_EXAMPLE_SLUGS = frozenset(
|
||||
{
|
||||
"image_to_image",
|
||||
"image_to_video",
|
||||
"speech_to_video",
|
||||
"text_to_audio",
|
||||
"text_to_image",
|
||||
"text_to_speech",
|
||||
"text_to_video",
|
||||
"x_to_text",
|
||||
"x_to_video_audio",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def load_model_display_names() -> dict[str, str]:
|
||||
try:
|
||||
@@ -46,6 +63,13 @@ def load_model_display_names() -> dict[str, str]:
|
||||
MODEL_DISPLAY_NAMES = load_model_display_names()
|
||||
|
||||
|
||||
def is_general_example(example: "Example") -> bool:
|
||||
"""Return whether an example should get a generated serving page."""
|
||||
if example.category not in SERVING_MODE_TITLES:
|
||||
return True
|
||||
return example.path.stem in GENERAL_EXAMPLE_SLUGS
|
||||
|
||||
|
||||
def fix_case(text: str) -> str:
|
||||
subs = {
|
||||
"api": "API",
|
||||
@@ -340,22 +364,21 @@ def update_nav_file(examples: list[Example]):
|
||||
examples_by_category[category] = []
|
||||
examples_by_category[category].append(example)
|
||||
|
||||
# Build the new Examples section - start with preserved items
|
||||
examples_section = preserved_items.copy()
|
||||
|
||||
# Add examples grouped by category, sorted by category name
|
||||
for category in sorted(examples_by_category.keys()):
|
||||
category_examples = sorted(examples_by_category[category], key=lambda e: e.path.stem)
|
||||
category_items = []
|
||||
for example in category_examples:
|
||||
def category_items(category: str) -> list[dict[str, str]]:
|
||||
items = []
|
||||
for example in sorted(examples_by_category[category], key=lambda e: e.path.stem):
|
||||
doc_path = EXAMPLE_DOC_DIR / example.category / f"{example.path.stem}.md"
|
||||
rel_path = doc_path.relative_to(ROOT_DIR / "docs")
|
||||
category_items.append({example.nav_title: rel_path.as_posix()})
|
||||
items.append({example.nav_title: rel_path.as_posix()})
|
||||
return items
|
||||
|
||||
if category_items:
|
||||
# Format category name (e.g., "offline_inference" -> "Offline Inference")
|
||||
# Build the new Examples section - start with preserved items.
|
||||
examples_section = preserved_items.copy()
|
||||
for category in sorted(examples_by_category.keys()):
|
||||
items = category_items(category)
|
||||
if items:
|
||||
category_title = fix_case(category.replace("_", " ").title())
|
||||
examples_section.append({category_title: category_items})
|
||||
examples_section.append({category_title: items})
|
||||
|
||||
# Update the nav structure
|
||||
nav_list[user_guide_idx]["User Guide"][examples_idx]["Examples"] = examples_section
|
||||
@@ -390,6 +413,8 @@ def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
|
||||
for path in category.glob("*/*.md"):
|
||||
examples.append(Example(path.parent, category.stem))
|
||||
|
||||
examples = [example for example in examples if is_general_example(example)]
|
||||
|
||||
# Generate the example documentation
|
||||
for example in sorted(examples, key=lambda e: e.path.stem):
|
||||
example_name = f"{example.path.stem}.md"
|
||||
|
||||
@@ -17,84 +17,93 @@ th {
|
||||
}
|
||||
</style>
|
||||
|
||||
| Architecture | Models | Example HF Models | NVIDIA GPU | AMD GPU | Ascend NPU | Intel GPU |
|
||||
|--------------|--------|-------------------|------------|---------|-----|-----------|
|
||||
| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | |
|
||||
| `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | |
|
||||
| `Gr00tN1d7Pipeline` | GR00T N1.7 | `nvidia/GR00T-N1.7-3B` | ✅︎ | | | |
|
||||
| `HunyuanImage3ForCausalMM` | HunyuanImage3.0 (DiT-only) | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2511 | `Qwen/Qwen-Image-Edit-2511` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `GlmImagePipeline` | GLM-Image | `zai-org/GLM-Image` | ✅︎ | ✅︎ | | |
|
||||
| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | |
|
||||
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | |
|
||||
| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
||||
| `LTX2DistilledPipeline` | LTX-2 distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
||||
| `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | |
|
||||
| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | | | |
|
||||
| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
|
||||
| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
|
||||
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit` | ✅︎ | | | |
|
||||
| `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | |
|
||||
| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | |
|
||||
| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | |
|
||||
| `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | |
|
||||
| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | |
|
||||
| `FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `SoulXSingerPipeline` | SoulX-Singer (SVS) | `Soul-AILab/SoulX-Singer` | ✅︎ | | | |
|
||||
| `SoulXSingerSVCPipeline` | SoulX-Singer-SVC | `Soul-AILab/SoulX-Singer` (`model-svc.pt`) | ✅︎ | | | |
|
||||
| `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `MingTTSForConditionalGeneration` | Ming-omni-tts dense 0.5B | `inclusionAI/Ming-omni-tts-0.5B` | ✅︎ | | | |
|
||||
| `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | |
|
||||
| `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | | |
|
||||
| `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | |
|
||||
| `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | |
|
||||
| `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | |
|
||||
| `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | |
|
||||
| `IndexTTS2TalkerForConditionalGeneration` | IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✅︎ | | | |
|
||||
| `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
|
||||
| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |
|
||||
| `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | |
|
||||
| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | |
|
||||
| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | |
|
||||
| `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | |
|
||||
| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | |
|
||||
| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
|
||||
| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
|
||||
| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
|
||||
|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
|
||||
| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | |
|
||||
| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
|`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | |
|
||||
| Architecture | Models | Example HF Models | NVIDIA GPU | AMD GPU | Ascend NPU | Intel GPU | Recipe |
|
||||
|--------------|--------|-------------------|------------|---------|-----|-----------|--------|
|
||||
| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen3-Omni.md) |
|
||||
| `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | | — |
|
||||
| `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | | — |
|
||||
| `Gr00tN1d7Pipeline` | GR00T N1.7 | `nvidia/GR00T-N1.7-3B` | ✅︎ | | | | — |
|
||||
| `HunyuanImage3ForCausalMM` | HunyuanImage3.0 (DiT-only) | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | | | | [Published](https://recipes.vllm.ai/Qwen/Qwen-Image) |
|
||||
| `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image-2512.md) |
|
||||
| `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2511 | `Qwen/Qwen-Image-Edit-2511` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `GlmImagePipeline` | GLM-Image | `zai-org/GLM-Image` | ✅︎ | ✅︎ | | | — |
|
||||
| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | | — |
|
||||
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | | — |
|
||||
| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | — |
|
||||
| `LTX2DistilledPipeline` | LTX-2 distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | | — |
|
||||
| `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | — |
|
||||
| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | ✅︎ | | | [Published](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) |
|
||||
| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | — |
|
||||
| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | | — |
|
||||
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit` | ✅︎ | | | | — |
|
||||
| `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | | — |
|
||||
| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | | — |
|
||||
| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
|
||||
| `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
|
||||
| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | | — |
|
||||
| `FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `SoulXSingerPipeline` | SoulX-Singer (SVS) | `Soul-AILab/SoulX-Singer` | ✅︎ | | | | — |
|
||||
| `SoulXSingerSVCPipeline` | SoulX-Singer-SVC | `Soul-AILab/SoulX-Singer` (`model-svc.pt`) | ✅︎ | | | | — |
|
||||
| `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | | — |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | | | | [Published](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `MingTTSForConditionalGeneration` | Ming-omni-tts dense 0.5B | `inclusionAI/Ming-omni-tts-0.5B` | ✅︎ | | | | — |
|
||||
| `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | | — |
|
||||
| `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | | | — |
|
||||
| `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | | — |
|
||||
| `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | | — |
|
||||
| `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | | — |
|
||||
| `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | | [Published](https://recipes.vllm.ai/bosonai/higgs-audio-v3-tts-4b) |
|
||||
| `IndexTTS2TalkerForConditionalGeneration` | IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✅︎ | | | | — |
|
||||
| `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | | — |
|
||||
| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | | — |
|
||||
| `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | | — |
|
||||
| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | | — |
|
||||
| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | | — |
|
||||
| `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | | — |
|
||||
| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | | — |
|
||||
| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | | — |
|
||||
| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
|
||||
| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
|
||||
|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | | — |
|
||||
| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | — |
|
||||
| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
|`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
|
||||
|
||||
The `Recipe` column links the corresponding deployment recipe. `Published` is
|
||||
preferred when a corresponding `recipes.vllm.ai` page exists; otherwise the
|
||||
link points to the repository recipe. `—` means that no recipe link has been
|
||||
added for that model row yet.
|
||||
|
||||
For rows with an audited recipe link, the hardware checkmarks reflect hardware
|
||||
explicitly documented in that recipe. Other rows retain the implementation
|
||||
support metadata until their recipe is audited.
|
||||
|
||||
✅︎ indicates the model is supported on that backend. Empty cells mean not listed as supported on that backend.
|
||||
|
||||
|
||||
@@ -1,247 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Verify a ModelOpt FP8 diffusers checkpoint exported by
|
||||
quantize_wan2_2_modelopt_fp8.py / quantize_hunyuanvideo_15_modelopt_fp8.py.
|
||||
|
||||
Three checks:
|
||||
A. transformer/config.json has a sane quantization_config block.
|
||||
B. transformer/*.safetensors contains FP8 (float8_e4m3fn) quantized tensors.
|
||||
C. transformer disk size is materially smaller than a BF16 baseline.
|
||||
|
||||
Example:
|
||||
python examples/quantization/check_modelopt_fp8_export.py \\
|
||||
--output ./hv15-480p-modelopt-fp8
|
||||
|
||||
# Optional: compare disk size against a local or HF BF16 baseline.
|
||||
python examples/quantization/check_modelopt_fp8_export.py \\
|
||||
--output ./hv15-480p-modelopt-fp8 \\
|
||||
--baseline hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
SUPPORTED_ALGOS = ("FP8",)
|
||||
|
||||
|
||||
def _check_config(transformer_dir: Path) -> tuple[int, str | None]:
|
||||
"""Returns (status, quant_algo). status: 0 pass, 1 fail, 2 warn."""
|
||||
cfg_path = transformer_dir / "config.json"
|
||||
if not cfg_path.exists():
|
||||
print(f"[FAIL] {cfg_path} missing.")
|
||||
return 1, None
|
||||
|
||||
with cfg_path.open(encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
|
||||
qc = cfg.get("quantization_config")
|
||||
if not isinstance(qc, dict):
|
||||
print(f"[FAIL] No `quantization_config` block in {cfg_path}.")
|
||||
return 1, None
|
||||
|
||||
print(f"[A] quantization_config from {cfg_path}:")
|
||||
print(json.dumps(qc, indent=2))
|
||||
|
||||
quant_algo = qc.get("quant_algo")
|
||||
issues = []
|
||||
if qc.get("quant_method") != "modelopt":
|
||||
issues.append(f"quant_method={qc.get('quant_method')!r} (expected 'modelopt')")
|
||||
if quant_algo not in SUPPORTED_ALGOS:
|
||||
issues.append(
|
||||
f"quant_algo={quant_algo!r} (expected one of {SUPPORTED_ALGOS} — vllm-omni adapter may not auto-detect)"
|
||||
)
|
||||
|
||||
if issues:
|
||||
print("[A] WARN — config looks incomplete:")
|
||||
for issue in issues:
|
||||
print(f" - {issue}")
|
||||
return 2, quant_algo
|
||||
print(f"[A] PASS — config looks correct (quant_algo={quant_algo}).")
|
||||
return 0, quant_algo
|
||||
|
||||
|
||||
def _read_safetensors_header(path: Path) -> dict:
|
||||
"""Read the JSON header of a safetensors file. Bypass-safe — doesn't materialize tensors.
|
||||
|
||||
Returns {tensor_name: {'dtype': 'F8_E4M3', 'shape': [...], 'data_offsets': [...]}}.
|
||||
Header dtype strings: F8_E4M3, F8_E5M2, BF16, F16, F32, F64, I8, I16, I32, I64, BOOL, U8, ...
|
||||
"""
|
||||
import struct
|
||||
|
||||
with open(path, "rb") as f:
|
||||
header_len = struct.unpack("<Q", f.read(8))[0]
|
||||
header = json.loads(f.read(header_len))
|
||||
header.pop("__metadata__", None)
|
||||
return header
|
||||
|
||||
|
||||
def _classify_weight_scale_granularity(weight_scale_shapes: list[list[int]]) -> str:
|
||||
"""Infer per-tensor vs per-channel vs per-block from sample weight_scale shapes.
|
||||
|
||||
ModelOpt block-wise produces shapes like `[16, 1, 16, 1]` (broadcasting dims of 1
|
||||
interleaved with block-count dims). We count "meaningful" dims — ones with size > 1 —
|
||||
and classify: 0 meaningful dims = per-tensor (scalar), 1 = per-channel, 2+ = per-block.
|
||||
"""
|
||||
if not weight_scale_shapes:
|
||||
return "no weight_scale tensors found"
|
||||
|
||||
def meaningful_dims(shape: list[int]) -> int:
|
||||
return sum(1 for d in shape if d > 1)
|
||||
|
||||
per_tensor = sum(1 for s in weight_scale_shapes if meaningful_dims(s) == 0)
|
||||
per_channel = sum(1 for s in weight_scale_shapes if meaningful_dims(s) == 1)
|
||||
per_block = sum(1 for s in weight_scale_shapes if meaningful_dims(s) >= 2)
|
||||
total = len(weight_scale_shapes)
|
||||
if per_tensor == total:
|
||||
return "per-tensor (all scalar scales)"
|
||||
if per_channel == total:
|
||||
return "per-channel (1 meaningful dim)"
|
||||
if per_block == total:
|
||||
return "per-block (2+ meaningful dims — e.g. [M//bm, 1, N//bn, 1] for tiles)"
|
||||
return f"mixed: per-tensor={per_tensor}, per-channel={per_channel}, per-block={per_block} of {total}"
|
||||
|
||||
|
||||
def _check_safetensors(transformer_dir: Path) -> int:
|
||||
"""Returns 0 on pass, 1 on fail. Reads on-disk dtype from the safetensors header."""
|
||||
files = sorted(transformer_dir.glob("*.safetensors"))
|
||||
if not files:
|
||||
print(f"[FAIL] No *.safetensors in {transformer_dir}.")
|
||||
return 1
|
||||
|
||||
header_dtype_counts: Counter[str] = Counter()
|
||||
sample_quant_weight_keys: list[str] = []
|
||||
sample_scale_keys: list[str] = []
|
||||
weight_scale_shapes: list[list[int]] = []
|
||||
sample_weight_scale_entries: list[tuple[str, list[int]]] = []
|
||||
|
||||
for f in files:
|
||||
try:
|
||||
header = _read_safetensors_header(f)
|
||||
except Exception as exc:
|
||||
print(f"[B] WARN — could not parse header of {f}: {exc}")
|
||||
continue
|
||||
for k, info in header.items():
|
||||
dtype = info.get("dtype", "?")
|
||||
header_dtype_counts[dtype] += 1
|
||||
# FP8 stores weights as F8_E4M3 directly.
|
||||
if dtype.startswith("F8") and k.endswith(".weight") and len(sample_quant_weight_keys) < 5:
|
||||
sample_quant_weight_keys.append(k)
|
||||
if k.endswith(("_scale", ".weight_scale", ".input_scale", "_scale_inv")) and len(sample_scale_keys) < 5:
|
||||
sample_scale_keys.append(k)
|
||||
if k.endswith(".weight_scale"):
|
||||
weight_scale_shapes.append(info.get("shape", []))
|
||||
if len(sample_weight_scale_entries) < 5:
|
||||
sample_weight_scale_entries.append((k, info.get("shape", [])))
|
||||
|
||||
print(f"\n[B] On-disk dtype counts across {len(files)} safetensors file(s) (from header, not get_tensor):")
|
||||
for dtype, count in sorted(header_dtype_counts.items(), key=lambda kv: -kv[1]):
|
||||
marker = " <-- FP8" if dtype.startswith("F8") else ""
|
||||
print(f" {dtype:10s} {count:>6d}{marker}")
|
||||
|
||||
quant_count = sum(c for d, c in header_dtype_counts.items() if d.startswith("F8"))
|
||||
if quant_count == 0:
|
||||
print("[B] FAIL — no FP8 tensors on disk. Calibration likely did not actually quantize the weights.")
|
||||
return 1
|
||||
|
||||
print(f"[B] PASS — {quant_count} FP8 tensors stored on disk.")
|
||||
if sample_quant_weight_keys:
|
||||
print(f" sample quantized weight tensors: {sample_quant_weight_keys[:3]}")
|
||||
if sample_scale_keys:
|
||||
print(f" sample scale tensors: {sample_scale_keys[:3]}")
|
||||
print(" (Note: torch's get_tensor() may return these as bf16 views on some versions —")
|
||||
print(" irrelevant; vLLM's loader uses native FP8 ops.)")
|
||||
|
||||
# Weight-scale granularity — per-tensor (scalar) vs per-channel (1-D) vs per-block (N-D).
|
||||
print(f"\n weight_scale granularity: {_classify_weight_scale_granularity(weight_scale_shapes)}")
|
||||
for key, shape in sample_weight_scale_entries[:3]:
|
||||
print(f" {key}: shape {shape}")
|
||||
return 0
|
||||
|
||||
|
||||
def _disk_size_gib(p: Path) -> float:
|
||||
return sum(f.stat().st_size for f in p.rglob("*") if f.is_file()) / (1024**3)
|
||||
|
||||
|
||||
def _check_size_vs_baseline(transformer_dir: Path, baseline: str | None) -> int:
|
||||
"""Returns 0 always (informational only)."""
|
||||
quant_size = _disk_size_gib(transformer_dir)
|
||||
print(f"\n[C] FP8 transformer disk size: {quant_size:.2f} GiB")
|
||||
|
||||
if baseline is None:
|
||||
print("[C] SKIP — pass --baseline <path or HF id> to compare against BF16.")
|
||||
return 0
|
||||
|
||||
baseline_path = Path(baseline)
|
||||
if not baseline_path.exists():
|
||||
# Try HF download.
|
||||
try:
|
||||
from huggingface_hub import snapshot_download
|
||||
except ImportError:
|
||||
print("[C] SKIP — huggingface_hub not installed and baseline not a local path.")
|
||||
return 0
|
||||
print(f" Downloading baseline transformer from HF: {baseline}")
|
||||
baseline_path = Path(snapshot_download(baseline, allow_patterns=["transformer/*"]))
|
||||
|
||||
bf16_dir = baseline_path / "transformer" if (baseline_path / "transformer").exists() else baseline_path
|
||||
bf16_size = _disk_size_gib(bf16_dir)
|
||||
if bf16_size == 0:
|
||||
print(f"[C] WARN — baseline transformer dir empty: {bf16_dir}")
|
||||
return 0
|
||||
|
||||
# Expected reduction for FP8: ~50%.
|
||||
min_reduction = 30
|
||||
reduction = (1 - quant_size / bf16_size) * 100
|
||||
print(f"[C] BF16 baseline transformer disk size: {bf16_size:.2f} GiB ({bf16_dir})")
|
||||
print(f"[C] Disk reduction: {reduction:.1f}% (FP8 transformer is {quant_size / bf16_size:.0%} of BF16)")
|
||||
if reduction < min_reduction:
|
||||
print(
|
||||
f"[C] WARN — FP8 should typically reduce disk by ~40-50%; <{min_reduction}% suggests partial quantization."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--output", required=True, help="Path to the exported ModelOpt FP8 checkpoint root.")
|
||||
p.add_argument(
|
||||
"--baseline",
|
||||
default=None,
|
||||
help="Optional BF16 baseline (local diffusers dir or HF id) for disk-size comparison.",
|
||||
)
|
||||
args = p.parse_args()
|
||||
|
||||
out_root = Path(args.output).expanduser().resolve()
|
||||
transformer_dir = out_root / "transformer"
|
||||
if not transformer_dir.exists():
|
||||
print(f"[FAIL] {transformer_dir} does not exist.")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Checking: {out_root}\n")
|
||||
|
||||
fail = 0
|
||||
config_status, _quant_algo = _check_config(transformer_dir)
|
||||
fail |= config_status
|
||||
fail |= _check_safetensors(transformer_dir)
|
||||
_check_size_vs_baseline(transformer_dir, args.baseline)
|
||||
|
||||
print()
|
||||
if fail == 0:
|
||||
print("=" * 60)
|
||||
print("ALL CHECKS PASSED — checkpoint looks ready for vllm-omni serving.")
|
||||
elif fail == 1:
|
||||
print("=" * 60)
|
||||
print("FAILURES detected — calibration may need to be re-run.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("=" * 60)
|
||||
print("WARNINGS only — checkpoint may serve but with caveats. See [A] above.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,448 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Quantize HunyuanVideo-1.5 (480p T2V) to a ModelOpt FP8 Hugging Face checkpoint.
|
||||
|
||||
Calibrates the DiT transformer using a small video prompt set and exports a
|
||||
diffusers-style directory whose transformer carries ModelOpt FP8 metadata.
|
||||
The exported checkpoint is consumable by vllm-omni's ModelOpt FP8 adapter
|
||||
(see vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py).
|
||||
|
||||
Layers kept full precision match the #2728 / #2795 pattern: modulation,
|
||||
AdaLayerNorm, entry/exit projections, embeddings, the token refiner path,
|
||||
and final proj_out. MHA quantizers are off by default; HV-1.5 self-attention
|
||||
empirically degrades under FP8 (see #2920 ablation).
|
||||
|
||||
Example:
|
||||
python examples/quantization/quantize_hunyuanvideo_15_modelopt_fp8.py \\
|
||||
--model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \\
|
||||
--output ./hv15-480p-modelopt-fp8 \\
|
||||
--overwrite
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from diffusers import DiffusionPipeline
|
||||
|
||||
DEFAULT_PROMPTS = [
|
||||
"A dog running across a field of golden wheat.",
|
||||
"An astronaut riding a horse across the surface of Mars, red dust swirling, cinematic wide shot.",
|
||||
"A hummingbird hovering in front of a vibrant red flower, slow motion, macro shot.",
|
||||
"A crackling campfire at night under a starry sky, sparks rising into the dark.",
|
||||
"An underwater shot of a coral reef with tropical fish swimming by, sun rays piercing the water.",
|
||||
"A close-up of a blooming rose covered in morning dew, soft natural light.",
|
||||
"A peaceful mountain village at dawn, mist rolling over the rooftops, cinematic establishing shot.",
|
||||
"A skateboarder doing a kickflip in an urban plaza, slow motion, golden hour lighting.",
|
||||
]
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--model", required=True, help="Input HV-1.5 diffusers directory or HF id.")
|
||||
p.add_argument("--output", required=True, help="Output directory for the ModelOpt FP8 checkpoint.")
|
||||
p.add_argument("--dtype", choices=("bfloat16", "float16"), default="bfloat16")
|
||||
p.add_argument("--height", type=int, default=480)
|
||||
p.add_argument("--width", type=int, default=832)
|
||||
p.add_argument(
|
||||
"--num-frames",
|
||||
type=int,
|
||||
default=33,
|
||||
help="Frames per calibration sample. 33 matches the typical short benchmark.",
|
||||
)
|
||||
p.add_argument("--guidance-scale", type=float, default=6.0)
|
||||
p.add_argument(
|
||||
"--calib-steps",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Denoising steps per calibration prompt (10 is enough for amax statistics).",
|
||||
)
|
||||
p.add_argument("--calib-size", type=int, default=8, help="How many prompts to use for calibration.")
|
||||
p.add_argument("--seed", type=int, default=42)
|
||||
p.add_argument(
|
||||
"--prompt",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Custom calibration prompt. Repeat to provide multiple.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--quantize-mha",
|
||||
action="store_true",
|
||||
help="Enable FP8 attention K/V/softmax quantizers. Off by default — empirically degrades HV-1.5 video output.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--weight-block-size",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Per-block weight quantization as 'M,N' (e.g. '128,128' for 128x128 tiles). "
|
||||
"Default: per-tensor (one scale per linear). Block-wise typically gives tighter quality at "
|
||||
"negligible memory cost. Static FP8 is exempt from upstream vLLM's online block-wise gate.",
|
||||
)
|
||||
p.add_argument("--overwrite", action="store_true", help="Replace an existing output directory.")
|
||||
return p
|
||||
|
||||
|
||||
def _parse_block_size(spec: str | None) -> list[int] | None:
|
||||
if spec is None:
|
||||
return None
|
||||
parts = [int(x) for x in spec.split(",") if x.strip()]
|
||||
if len(parts) != 2:
|
||||
raise SystemExit(f"--weight-block-size must be 'M,N' (2 ints), got {spec!r}")
|
||||
return parts
|
||||
|
||||
|
||||
def _require_modelopt() -> Any:
|
||||
try:
|
||||
import modelopt.torch.quantization as mtq
|
||||
except ModuleNotFoundError as exc:
|
||||
raise SystemExit(
|
||||
"NVIDIA ModelOpt is not installed. Install with:\n"
|
||||
" pip install 'nvidia-modelopt[all]'\n"
|
||||
f"Original error: {exc}"
|
||||
) from exc
|
||||
return mtq
|
||||
|
||||
|
||||
def _ensure_paths(args: argparse.Namespace) -> tuple[str, Path]:
|
||||
model_path = args.model
|
||||
output_dir = Path(args.output).expanduser().resolve()
|
||||
if output_dir.exists():
|
||||
if not args.overwrite:
|
||||
raise SystemExit(f"Output directory already exists: {output_dir}\nPass --overwrite to replace it.")
|
||||
shutil.rmtree(output_dir)
|
||||
return model_path, output_dir
|
||||
|
||||
|
||||
def _select_dtype(name: str) -> torch.dtype:
|
||||
return {"bfloat16": torch.bfloat16, "float16": torch.float16}[name]
|
||||
|
||||
|
||||
def _build_prompts(args: argparse.Namespace) -> list[str]:
|
||||
prompts = args.prompt or DEFAULT_PROMPTS
|
||||
if args.calib_size <= 0:
|
||||
raise SystemExit("--calib-size must be positive.")
|
||||
if len(prompts) < args.calib_size:
|
||||
repeats = (args.calib_size + len(prompts) - 1) // len(prompts)
|
||||
prompts = (prompts * repeats)[: args.calib_size]
|
||||
return prompts[: args.calib_size]
|
||||
|
||||
|
||||
# Layers to KEEP at full precision (mirror of the #2920 wiring + #2728/#2795 skip pattern).
|
||||
# - x_embedder, image_embedder, context_embedder*, time_embed*, cond_type_embed: entry/embedding
|
||||
# - norm_out, norm1*.linear, norm1_context*.linear, norm2*, norm2_context*: AdaLayerNorm modulation
|
||||
# - proj_out: final output projection
|
||||
# - token_refiner*: text-encoder refinement uses diffusers raw nn.Linear
|
||||
def _filter_func_hv15(name: str) -> bool:
|
||||
pattern = re.compile(
|
||||
r"(proj_out.*|"
|
||||
r".*(x_embedder|image_embedder|context_embedder|context_embedder_2|"
|
||||
r"time_embed|cond_type_embed|"
|
||||
r"norm_out|norm1\.linear|norm1_context\.linear|norm2|norm2_context|"
|
||||
r"token_refiner).*)"
|
||||
)
|
||||
return pattern.match(name) is not None
|
||||
|
||||
|
||||
def _mha_filter_func(name: str) -> bool:
|
||||
pattern = re.compile(
|
||||
r".*(q_bmm_quantizer|k_bmm_quantizer|v_bmm_quantizer|softmax_quantizer|bmm2_output_quantizer).*"
|
||||
)
|
||||
return pattern.match(name) is not None
|
||||
|
||||
|
||||
def _disable_known_problematic_quantizers(mtq: Any, backbone: torch.nn.Module, *, quantize_mha: bool) -> None:
|
||||
if not hasattr(mtq, "disable_quantizer"):
|
||||
return
|
||||
mtq.disable_quantizer(backbone, _filter_func_hv15)
|
||||
if not quantize_mha:
|
||||
mtq.disable_quantizer(backbone, _mha_filter_func)
|
||||
|
||||
|
||||
def _load_pipeline(model_path: str, dtype: torch.dtype) -> DiffusionPipeline:
|
||||
pipe = DiffusionPipeline.from_pretrained(model_path, torch_dtype=dtype)
|
||||
if hasattr(pipe, "set_progress_bar_config"):
|
||||
pipe.set_progress_bar_config(disable=True)
|
||||
pipe.to("cuda")
|
||||
return pipe
|
||||
|
||||
|
||||
def _build_forward_loop(pipe: DiffusionPipeline, args: argparse.Namespace, prompts: list[str]):
|
||||
generator = torch.Generator(device="cuda")
|
||||
|
||||
# Try to set guidance on the pipeline's guider object up front (modern
|
||||
# diffusers HV-1.5 uses a Guider abstraction, not a per-call kwarg). Falls
|
||||
# back silently — calibration uses whatever default the pipeline ships with.
|
||||
guider = getattr(pipe, "guider", None)
|
||||
if guider is not None and hasattr(guider, "guidance_scale"):
|
||||
try:
|
||||
guider.guidance_scale = args.guidance_scale
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
base_kwargs = dict(
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
num_frames=args.num_frames,
|
||||
num_inference_steps=args.calib_steps,
|
||||
output_type="latent",
|
||||
)
|
||||
|
||||
def forward_loop(*_unused_args, **_unused_kwargs) -> None:
|
||||
with torch.inference_mode():
|
||||
for idx, prompt in enumerate(prompts):
|
||||
generator.manual_seed(args.seed + idx)
|
||||
# Try with guidance_scale first; fall back without on TypeError
|
||||
# for pipelines (like HV-1.5) that take CFG via guider config.
|
||||
try:
|
||||
pipe(prompt=prompt, generator=generator, guidance_scale=args.guidance_scale, **base_kwargs)
|
||||
except TypeError as exc:
|
||||
if "guidance_scale" not in str(exc):
|
||||
raise
|
||||
pipe(prompt=prompt, generator=generator, **base_kwargs)
|
||||
|
||||
return forward_loop
|
||||
|
||||
|
||||
def _summarize_export(output_dir: Path) -> None:
|
||||
cfg_path = output_dir / "transformer" / "config.json"
|
||||
if not cfg_path.exists():
|
||||
print(f"[warn] {cfg_path} missing.", file=sys.stderr)
|
||||
return
|
||||
with cfg_path.open(encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
qc = cfg.get("quantization_config")
|
||||
if not isinstance(qc, dict):
|
||||
print("[warn] No quantization_config in transformer/config.json.", file=sys.stderr)
|
||||
return
|
||||
print("Export summary:")
|
||||
print(f" quant_method: {qc.get('quant_method')}")
|
||||
print(f" quant_algo: {qc.get('quant_algo')}")
|
||||
producer = qc.get("producer")
|
||||
if isinstance(producer, dict):
|
||||
print(f" producer: {producer.get('name')} {producer.get('version')}")
|
||||
print(f" config path: {cfg_path}")
|
||||
|
||||
|
||||
def _force_export_quantized_weights(backbone: torch.nn.Module, dtype: torch.dtype) -> int:
|
||||
"""Convert in-memory weights of quantized modules to actual FP8 storage.
|
||||
|
||||
`export_hf_checkpoint` skips this step for unknown model types (HV-1.5 isn't
|
||||
in ModelOpt's recognized-model registry), so we must call the per-weight
|
||||
export helper ourselves. Same workaround as the HunyuanImage-3 calibration
|
||||
helper.
|
||||
"""
|
||||
from modelopt.torch.export.quant_utils import (
|
||||
QUANTIZATION_NONE,
|
||||
get_quantization_format,
|
||||
quantizer_attr_names,
|
||||
weight_attr_names,
|
||||
)
|
||||
from modelopt.torch.export.unified_export_hf import _export_quantized_weight
|
||||
|
||||
exported = 0
|
||||
for name, module in backbone.named_modules():
|
||||
try:
|
||||
quantization_format = get_quantization_format(module)
|
||||
except Exception as exc:
|
||||
print(f"[warn] Could not inspect quantization format for {name}: {exc}", file=sys.stderr)
|
||||
continue
|
||||
if quantization_format == QUANTIZATION_NONE:
|
||||
continue
|
||||
for weight_name in weight_attr_names(module):
|
||||
quantizer_attrs = quantizer_attr_names(weight_name)
|
||||
weight_quantizer = getattr(module, quantizer_attrs.weight_quantizer, None)
|
||||
if weight_quantizer is None or not getattr(weight_quantizer, "is_enabled", False):
|
||||
continue
|
||||
_export_quantized_weight(module, dtype, weight_name)
|
||||
exported += 1
|
||||
return exported
|
||||
|
||||
|
||||
def _hv15_quant_config_block(weight_block_size: list[int] | None = None) -> dict:
|
||||
"""Mirror ModelOpt FP8 metadata expected by vllm-omni's adapter (#2913).
|
||||
|
||||
Same shape as the HunyuanImage-3 author's _hunyuan_quant_config(). When
|
||||
`weight_block_size` is given, advertise block-wise weight quantization in
|
||||
the saved metadata (so consumers know to expect multi-element scale tensors).
|
||||
"""
|
||||
weights_cfg: dict = {"dynamic": False, "num_bits": 8, "type": "float"}
|
||||
if weight_block_size is not None:
|
||||
weights_cfg["strategy"] = "block"
|
||||
weights_cfg["block_structure"] = f"{weight_block_size[0]}x{weight_block_size[1]}"
|
||||
return {
|
||||
"config_groups": {
|
||||
"group_0": {
|
||||
"input_activations": {"dynamic": False, "num_bits": 8, "type": "float"},
|
||||
"weights": weights_cfg,
|
||||
"targets": ["Linear"],
|
||||
}
|
||||
},
|
||||
"ignore": [
|
||||
"context_embedder*",
|
||||
"context_embedder_2*",
|
||||
"cond_type_embed*",
|
||||
"image_embedder*",
|
||||
"norm1.linear*",
|
||||
"norm1_context.linear*",
|
||||
"norm2*",
|
||||
"norm2_context*",
|
||||
"norm_out*",
|
||||
"proj_out*",
|
||||
"time_embed*",
|
||||
"token_refiner*",
|
||||
"x_embedder*",
|
||||
],
|
||||
"producer": {"name": "modelopt"},
|
||||
"quant_algo": "FP8",
|
||||
"quant_method": "modelopt",
|
||||
}
|
||||
|
||||
|
||||
def _patch_quant_config(output_dir: Path, weight_block_size: list[int] | None = None) -> None:
|
||||
"""Inject quant_algo: FP8 + config_groups into transformer/config.json so
|
||||
vllm-omni's adapter (#2913) recognises the checkpoint as ModelOpt FP8."""
|
||||
cfg_path = output_dir / "transformer" / "config.json"
|
||||
with cfg_path.open(encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
|
||||
new_qc = _hv15_quant_config_block(weight_block_size=weight_block_size)
|
||||
existing = cfg.get("quantization_config")
|
||||
if isinstance(existing, dict):
|
||||
producer = existing.get("producer")
|
||||
if isinstance(producer, dict):
|
||||
new_qc["producer"] = producer
|
||||
|
||||
cfg["quantization_config"] = new_qc
|
||||
with cfg_path.open("w", encoding="utf-8") as f:
|
||||
json.dump(cfg, f, indent=2)
|
||||
|
||||
|
||||
def _save_pipeline_with_fp8_transformer(
|
||||
pipe: DiffusionPipeline,
|
||||
model_path: str,
|
||||
output_dir: Path,
|
||||
max_shard_size: str = "5GB",
|
||||
) -> None:
|
||||
"""Save the pipeline with the (now FP8) transformer.
|
||||
|
||||
Copies the source directory verbatim except for `transformer/`, then
|
||||
saves the transformer with quantizers hidden so the state dict contains
|
||||
only the FP8 weights + scale tensors.
|
||||
"""
|
||||
from modelopt.torch.export.diffusers_utils import hide_quantizers_from_state_dict
|
||||
|
||||
src = Path(model_path)
|
||||
if not src.exists():
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
src = Path(snapshot_download(model_path))
|
||||
|
||||
if output_dir.exists():
|
||||
shutil.rmtree(output_dir)
|
||||
shutil.copytree(src, output_dir, ignore=shutil.ignore_patterns("transformer"))
|
||||
|
||||
transformer_out = output_dir / "transformer"
|
||||
# `hide_quantizers_from_state_dict` walks named_modules(); pass the actual
|
||||
# nn.Module (transformer), not the diffusers Pipeline wrapper.
|
||||
with hide_quantizers_from_state_dict(pipe.transformer):
|
||||
pipe.transformer.save_pretrained(
|
||||
str(transformer_out),
|
||||
safe_serialization=True,
|
||||
max_shard_size=max_shard_size,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = _build_parser().parse_args()
|
||||
if not torch.cuda.is_available():
|
||||
raise SystemExit("CUDA is required for ModelOpt FP8 quantization.")
|
||||
|
||||
mtq = _require_modelopt()
|
||||
model_path, output_dir = _ensure_paths(args)
|
||||
dtype = _select_dtype(args.dtype)
|
||||
prompts = _build_prompts(args)
|
||||
|
||||
print("Quantization plan:")
|
||||
weight_block_size = _parse_block_size(args.weight_block_size)
|
||||
|
||||
print(f" input: {args.model}")
|
||||
print(f" output: {output_dir}")
|
||||
print(f" dtype: {dtype}")
|
||||
print(f" height/width: {args.height}x{args.width}")
|
||||
print(f" num_frames: {args.num_frames}")
|
||||
print(f" calib_size: {len(prompts)}")
|
||||
print(f" calib_steps: {args.calib_steps}")
|
||||
print(f" quantize_mha: {args.quantize_mha}")
|
||||
print(
|
||||
f" weight strategy: {'block-wise ' + str(weight_block_size) if weight_block_size else 'per-tensor (default)'}"
|
||||
)
|
||||
|
||||
pipe = _load_pipeline(model_path, dtype)
|
||||
backbone = pipe.transformer
|
||||
|
||||
quant_config = copy.deepcopy(mtq.FP8_DEFAULT_CFG)
|
||||
if weight_block_size is not None:
|
||||
# Switch from per-tensor (default) to block-wise weight quantization.
|
||||
# ModelOpt's wildcard "*weight_quantizer" matches every linear's weight quantizer.
|
||||
quant_config["quant_cfg"]["*weight_quantizer"] = {
|
||||
"num_bits": (4, 3), # E4M3 (FP8 weights, same as default)
|
||||
"block_sizes": {-1: weight_block_size[1], -2: weight_block_size[0]},
|
||||
}
|
||||
print(
|
||||
f" -> overriding weight quantizer with block_sizes={weight_block_size} "
|
||||
f"({weight_block_size[0]}x{weight_block_size[1]} tiles)"
|
||||
)
|
||||
|
||||
forward_loop = _build_forward_loop(pipe, args, prompts)
|
||||
quantized = mtq.quantize(backbone, quant_config, forward_loop)
|
||||
if quantized is not None:
|
||||
pipe.transformer = quantized
|
||||
backbone = quantized
|
||||
|
||||
_disable_known_problematic_quantizers(mtq, backbone, quantize_mha=args.quantize_mha)
|
||||
|
||||
print("\nForcing FP8 weight serialization (HV-1.5 isn't in ModelOpt's recognized-model registry,")
|
||||
print("so we have to call the per-weight export helper ourselves)...")
|
||||
exported = _force_export_quantized_weights(backbone, dtype)
|
||||
print(f" -> {exported} weights converted to FP8 in memory")
|
||||
if exported == 0:
|
||||
raise SystemExit(
|
||||
"No quantized weights were exported. Calibration may have skipped every layer "
|
||||
"(check the disable_quantizer regex) or `mtq.quantize` did not actually wrap any "
|
||||
"weight quantizers."
|
||||
)
|
||||
|
||||
print("\nSaving pipeline with FP8 transformer...")
|
||||
_save_pipeline_with_fp8_transformer(pipe, model_path, output_dir)
|
||||
_patch_quant_config(output_dir, weight_block_size=weight_block_size)
|
||||
print(f"Saved to: {output_dir}")
|
||||
_summarize_export(output_dir)
|
||||
|
||||
print("\nNext: validate the checkpoint with vllm-omni:")
|
||||
print(
|
||||
" python examples/offline_inference/text_to_video/text_to_video.py \\\n"
|
||||
f" --model {output_dir} \\\n"
|
||||
" --quantization fp8 \\\n"
|
||||
" --prompt 'A dog running across a field of golden wheat.' \\\n"
|
||||
f" --height {args.height} --width {args.width} --num-frames {args.num_frames} \\\n"
|
||||
" --num-inference-steps 30 --guidance-scale 6.0 --seed 42 \\\n"
|
||||
" --output outputs/hv15_modelopt_fp8.mp4 \\\n"
|
||||
" --enforce-eager"
|
||||
)
|
||||
print(
|
||||
"\n (--quantization fp8 is auto-upgraded to ModelOpt FP8 at runtime because the "
|
||||
"checkpoint's config.json has modelopt metadata.)"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,471 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Quantize Wan2.2 (TI2V-5B, 704x1280 T2V) to a ModelOpt FP8 Hugging Face checkpoint.
|
||||
|
||||
Calibrates the DiT transformer using a small video prompt set and exports a
|
||||
diffusers-style directory whose transformer carries ModelOpt FP8 metadata.
|
||||
The exported checkpoint is consumable by vllm-omni's ModelOpt FP8 adapter
|
||||
(see vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py).
|
||||
|
||||
Layers kept full precision match the #2728 / #2795 pattern: condition embedder
|
||||
(time/text/image), patch embedding, modulation (scale_shift_table), final
|
||||
norm + proj_out, and sequence-parallel helpers. All attention + FFN linears
|
||||
are quantized — static calibration handles the numerics that online FP8
|
||||
couldn't (see #2920 ablation).
|
||||
|
||||
Default target is `Wan-AI/Wan2.2-TI2V-5B-Diffusers`, the dense 5B variant that
|
||||
fits 80GB BF16. The A14B MoE variants need 2+ GPUs and are out of scope here.
|
||||
|
||||
Example:
|
||||
python examples/quantization/quantize_wan2_2_modelopt_fp8.py \\
|
||||
--model Wan-AI/Wan2.2-TI2V-5B-Diffusers \\
|
||||
--output ./wan22-ti2v-modelopt-fp8 \\
|
||||
--overwrite
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from diffusers import DiffusionPipeline
|
||||
|
||||
DEFAULT_PROMPTS = [
|
||||
"A dog running across a field of golden wheat.",
|
||||
"An astronaut riding a horse across the surface of Mars, red dust swirling, cinematic wide shot.",
|
||||
"A hummingbird hovering in front of a vibrant red flower, slow motion, macro shot.",
|
||||
"A crackling campfire at night under a starry sky, sparks rising into the dark.",
|
||||
"An underwater shot of a coral reef with tropical fish swimming by, sun rays piercing the water.",
|
||||
"A close-up of a blooming rose covered in morning dew, soft natural light.",
|
||||
"A peaceful mountain village at dawn, mist rolling over the rooftops, cinematic establishing shot.",
|
||||
"A skateboarder doing a kickflip in an urban plaza, slow motion, golden hour lighting.",
|
||||
]
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--model", required=True, help="Input Wan2.2 diffusers directory or HF id.")
|
||||
p.add_argument("--output", required=True, help="Output directory for the ModelOpt FP8 checkpoint.")
|
||||
p.add_argument("--dtype", choices=("bfloat16", "float16"), default="bfloat16")
|
||||
p.add_argument("--height", type=int, default=704, help="Calibration video height (Wan2.2 TI2V-5B native: 704).")
|
||||
p.add_argument("--width", type=int, default=1280, help="Calibration video width (Wan2.2 TI2V-5B native: 1280).")
|
||||
p.add_argument(
|
||||
"--num-frames",
|
||||
type=int,
|
||||
default=49,
|
||||
help="Frames per calibration sample. 49 matches the typical short benchmark; "
|
||||
"use 17 to reduce memory pressure during calibration.",
|
||||
)
|
||||
p.add_argument("--guidance-scale", type=float, default=5.0)
|
||||
p.add_argument(
|
||||
"--calib-steps",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Denoising steps per calibration prompt (10 is enough for amax statistics).",
|
||||
)
|
||||
p.add_argument("--calib-size", type=int, default=8, help="How many prompts to use for calibration.")
|
||||
p.add_argument("--seed", type=int, default=42)
|
||||
p.add_argument(
|
||||
"--prompt",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Custom calibration prompt. Repeat to provide multiple.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--quantize-mha",
|
||||
action="store_true",
|
||||
help="Enable FP8 attention K/V/softmax quantizers. Off by default — Wan2.2's long attention "
|
||||
"sequences amplified FP8 drift in the online ablation (see #2920).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--weight-block-size",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Per-block weight quantization as 'M,N' (e.g. '128,128'). Default per-tensor. "
|
||||
"Note: vllm-omni's ModelOpt adapter may not yet dispatch block-wise scales — check #2924 "
|
||||
"for the HV-1.5 investigation status before relying on this.",
|
||||
)
|
||||
p.add_argument("--overwrite", action="store_true", help="Replace an existing output directory.")
|
||||
return p
|
||||
|
||||
|
||||
def _parse_block_size(spec: str | None) -> list[int] | None:
|
||||
if spec is None:
|
||||
return None
|
||||
parts = [int(x) for x in spec.split(",") if x.strip()]
|
||||
if len(parts) != 2:
|
||||
raise SystemExit(f"--weight-block-size must be 'M,N' (2 ints), got {spec!r}")
|
||||
return parts
|
||||
|
||||
|
||||
def _require_modelopt() -> Any:
|
||||
try:
|
||||
import modelopt.torch.quantization as mtq
|
||||
except ModuleNotFoundError as exc:
|
||||
raise SystemExit(
|
||||
"NVIDIA ModelOpt is not installed. Install with:\n"
|
||||
" pip install 'nvidia-modelopt[all]'\n"
|
||||
f"Original error: {exc}"
|
||||
) from exc
|
||||
return mtq
|
||||
|
||||
|
||||
def _ensure_paths(args: argparse.Namespace) -> tuple[str, Path]:
|
||||
model_path = args.model
|
||||
output_dir = Path(args.output).expanduser().resolve()
|
||||
if output_dir.exists():
|
||||
if not args.overwrite:
|
||||
raise SystemExit(f"Output directory already exists: {output_dir}\nPass --overwrite to replace it.")
|
||||
shutil.rmtree(output_dir)
|
||||
return model_path, output_dir
|
||||
|
||||
|
||||
def _select_dtype(name: str) -> torch.dtype:
|
||||
return {"bfloat16": torch.bfloat16, "float16": torch.float16}[name]
|
||||
|
||||
|
||||
def _build_prompts(args: argparse.Namespace) -> list[str]:
|
||||
prompts = args.prompt or DEFAULT_PROMPTS
|
||||
if args.calib_size <= 0:
|
||||
raise SystemExit("--calib-size must be positive.")
|
||||
if len(prompts) < args.calib_size:
|
||||
repeats = (args.calib_size + len(prompts) - 1) // len(prompts)
|
||||
prompts = (prompts * repeats)[: args.calib_size]
|
||||
return prompts[: args.calib_size]
|
||||
|
||||
|
||||
# Layers to KEEP at full precision. Wan2.2's module naming:
|
||||
# - condition_embedder: time_embedder, time_proj, text_embedder, image_embedder (I2V)
|
||||
# - patch_embedding: Conv3dLayer (already not Linear, belt-and-suspenders skip)
|
||||
# - scale_shift_table: nn.Parameter modulation (not Linear, but pattern guard)
|
||||
# - norm_out: AdaLayerNorm final
|
||||
# - proj_out: final nn.Linear
|
||||
# - timestep_proj_prepare / output_scale_shift_prepare: SP helpers
|
||||
def _filter_func_wan22(name: str) -> bool:
|
||||
pattern = re.compile(
|
||||
r"(proj_out.*|"
|
||||
r".*(condition_embedder|patch_embedding|"
|
||||
r"norm_out|scale_shift_table|"
|
||||
r"timestep_proj_prepare|output_scale_shift_prepare).*)"
|
||||
)
|
||||
return pattern.match(name) is not None
|
||||
|
||||
|
||||
def _mha_filter_func(name: str) -> bool:
|
||||
pattern = re.compile(
|
||||
r".*(q_bmm_quantizer|k_bmm_quantizer|v_bmm_quantizer|softmax_quantizer|bmm2_output_quantizer).*"
|
||||
)
|
||||
return pattern.match(name) is not None
|
||||
|
||||
|
||||
def _disable_known_problematic_quantizers(mtq: Any, backbone: torch.nn.Module, *, quantize_mha: bool) -> None:
|
||||
if not hasattr(mtq, "disable_quantizer"):
|
||||
return
|
||||
mtq.disable_quantizer(backbone, _filter_func_wan22)
|
||||
if not quantize_mha:
|
||||
mtq.disable_quantizer(backbone, _mha_filter_func)
|
||||
|
||||
|
||||
def _load_pipeline(model_path: str, dtype: torch.dtype) -> DiffusionPipeline:
|
||||
pipe = DiffusionPipeline.from_pretrained(model_path, torch_dtype=dtype)
|
||||
if hasattr(pipe, "set_progress_bar_config"):
|
||||
pipe.set_progress_bar_config(disable=True)
|
||||
pipe.to("cuda")
|
||||
return pipe
|
||||
|
||||
|
||||
def _build_forward_loop(pipe: DiffusionPipeline, args: argparse.Namespace, prompts: list[str]):
|
||||
generator = torch.Generator(device="cuda")
|
||||
|
||||
# Try setting guidance on the pipeline's guider if present (newer diffusers APIs).
|
||||
guider = getattr(pipe, "guider", None)
|
||||
if guider is not None and hasattr(guider, "guidance_scale"):
|
||||
try:
|
||||
guider.guidance_scale = args.guidance_scale
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
base_kwargs = dict(
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
num_frames=args.num_frames,
|
||||
num_inference_steps=args.calib_steps,
|
||||
output_type="latent",
|
||||
)
|
||||
|
||||
def forward_loop(*_unused_args, **_unused_kwargs) -> None:
|
||||
with torch.inference_mode():
|
||||
for idx, prompt in enumerate(prompts):
|
||||
generator.manual_seed(args.seed + idx)
|
||||
# Try with guidance_scale first; fall back without on TypeError
|
||||
# for pipelines that take CFG via guider config only.
|
||||
try:
|
||||
pipe(prompt=prompt, generator=generator, guidance_scale=args.guidance_scale, **base_kwargs)
|
||||
except TypeError as exc:
|
||||
if "guidance_scale" not in str(exc):
|
||||
raise
|
||||
pipe(prompt=prompt, generator=generator, **base_kwargs)
|
||||
|
||||
return forward_loop
|
||||
|
||||
|
||||
def _summarize_export(output_dir: Path) -> None:
|
||||
cfg_path = output_dir / "transformer" / "config.json"
|
||||
if not cfg_path.exists():
|
||||
print(f"[warn] {cfg_path} missing.", file=sys.stderr)
|
||||
return
|
||||
with cfg_path.open(encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
qc = cfg.get("quantization_config")
|
||||
if not isinstance(qc, dict):
|
||||
print("[warn] No quantization_config in transformer/config.json.", file=sys.stderr)
|
||||
return
|
||||
print("Export summary:")
|
||||
print(f" quant_method: {qc.get('quant_method')}")
|
||||
print(f" quant_algo: {qc.get('quant_algo')}")
|
||||
producer = qc.get("producer")
|
||||
if isinstance(producer, dict):
|
||||
print(f" producer: {producer.get('name')} {producer.get('version')}")
|
||||
print(f" config path: {cfg_path}")
|
||||
|
||||
|
||||
def _force_export_quantized_weights(backbone: torch.nn.Module, dtype: torch.dtype) -> int:
|
||||
"""Convert in-memory weights of quantized modules to actual FP8 storage.
|
||||
|
||||
`export_hf_checkpoint` skips this step for unknown model types (Wan2.2 isn't
|
||||
in ModelOpt's recognized-model registry), so we must call the per-weight
|
||||
export helper ourselves. Same workaround as the HunyuanVideo-1.5 / HunyuanImage-3
|
||||
calibration helpers.
|
||||
"""
|
||||
from modelopt.torch.export.quant_utils import (
|
||||
QUANTIZATION_NONE,
|
||||
get_quantization_format,
|
||||
quantizer_attr_names,
|
||||
weight_attr_names,
|
||||
)
|
||||
from modelopt.torch.export.unified_export_hf import _export_quantized_weight
|
||||
|
||||
exported = 0
|
||||
for name, module in backbone.named_modules():
|
||||
try:
|
||||
quantization_format = get_quantization_format(module)
|
||||
except Exception as exc:
|
||||
print(f"[warn] Could not inspect quantization format for {name}: {exc}", file=sys.stderr)
|
||||
continue
|
||||
if quantization_format == QUANTIZATION_NONE:
|
||||
continue
|
||||
for weight_name in weight_attr_names(module):
|
||||
quantizer_attrs = quantizer_attr_names(weight_name)
|
||||
weight_quantizer = getattr(module, quantizer_attrs.weight_quantizer, None)
|
||||
if weight_quantizer is None or not getattr(weight_quantizer, "is_enabled", False):
|
||||
continue
|
||||
_export_quantized_weight(module, dtype, weight_name)
|
||||
exported += 1
|
||||
return exported
|
||||
|
||||
|
||||
def _wan22_quant_config_block(weight_block_size: list[int] | None = None) -> dict:
|
||||
"""Mirror ModelOpt FP8 metadata expected by vllm-omni's adapter (#2913)."""
|
||||
weights_cfg: dict = {"dynamic": False, "num_bits": 8, "type": "float"}
|
||||
if weight_block_size is not None:
|
||||
weights_cfg["strategy"] = "block"
|
||||
weights_cfg["block_structure"] = f"{weight_block_size[0]}x{weight_block_size[1]}"
|
||||
return {
|
||||
"config_groups": {
|
||||
"group_0": {
|
||||
"input_activations": {"dynamic": False, "num_bits": 8, "type": "float"},
|
||||
"weights": weights_cfg,
|
||||
"targets": ["Linear"],
|
||||
}
|
||||
},
|
||||
"ignore": [
|
||||
"condition_embedder*",
|
||||
"norm_out*",
|
||||
"output_scale_shift_prepare*",
|
||||
"patch_embedding*",
|
||||
"proj_out*",
|
||||
"scale_shift_table*",
|
||||
"timestep_proj_prepare*",
|
||||
],
|
||||
"producer": {"name": "modelopt"},
|
||||
"quant_algo": "FP8",
|
||||
"quant_method": "modelopt",
|
||||
}
|
||||
|
||||
|
||||
def _patch_quant_config(output_dir: Path, weight_block_size: list[int] | None = None) -> None:
|
||||
"""Inject quant_algo: FP8 + config_groups into each transformer's config.json
|
||||
so vllm-omni's adapter (#2913) recognises the checkpoint as ModelOpt FP8.
|
||||
|
||||
Patches both ``transformer`` and ``transformer_2`` (A14B) when present."""
|
||||
for sub in ("transformer", "transformer_2"):
|
||||
cfg_path = output_dir / sub / "config.json"
|
||||
if not cfg_path.exists():
|
||||
continue
|
||||
with cfg_path.open(encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
|
||||
new_qc = _wan22_quant_config_block(weight_block_size=weight_block_size)
|
||||
existing = cfg.get("quantization_config")
|
||||
if isinstance(existing, dict):
|
||||
producer = existing.get("producer")
|
||||
if isinstance(producer, dict):
|
||||
new_qc["producer"] = producer
|
||||
|
||||
cfg["quantization_config"] = new_qc
|
||||
with cfg_path.open("w", encoding="utf-8") as f:
|
||||
json.dump(cfg, f, indent=2)
|
||||
|
||||
|
||||
def _list_transformers(pipe: DiffusionPipeline) -> list[tuple[str, torch.nn.Module]]:
|
||||
"""Return [(attr_name, module)] for every DiT on the pipeline.
|
||||
|
||||
Wan2.2 A14B has both ``transformer`` (low-noise) and ``transformer_2``
|
||||
(high-noise). TI2V-5B has only ``transformer``.
|
||||
"""
|
||||
out = []
|
||||
for attr in ("transformer", "transformer_2"):
|
||||
mod = getattr(pipe, attr, None)
|
||||
if mod is not None:
|
||||
out.append((attr, mod))
|
||||
return out
|
||||
|
||||
|
||||
def _save_pipeline_with_fp8_transformer(
|
||||
pipe: DiffusionPipeline,
|
||||
model_path: str,
|
||||
output_dir: Path,
|
||||
max_shard_size: str = "5GB",
|
||||
) -> None:
|
||||
"""Copy source dir verbatim minus transformer dirs, then save each quantized DiT.
|
||||
|
||||
Handles A14B's dual transformer (``transformer`` + ``transformer_2``) as well
|
||||
as the single-DiT TI2V-5B.
|
||||
"""
|
||||
from modelopt.torch.export.diffusers_utils import hide_quantizers_from_state_dict
|
||||
|
||||
src = Path(model_path)
|
||||
if not src.exists():
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
src = Path(snapshot_download(model_path))
|
||||
|
||||
if output_dir.exists():
|
||||
shutil.rmtree(output_dir)
|
||||
shutil.copytree(src, output_dir, ignore=shutil.ignore_patterns("transformer", "transformer_2"))
|
||||
|
||||
for attr, backbone in _list_transformers(pipe):
|
||||
transformer_out = output_dir / attr
|
||||
# Pass the nn.Module (transformer), not the Pipeline wrapper.
|
||||
with hide_quantizers_from_state_dict(backbone):
|
||||
backbone.save_pretrained(
|
||||
str(transformer_out),
|
||||
safe_serialization=True,
|
||||
max_shard_size=max_shard_size,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = _build_parser().parse_args()
|
||||
if not torch.cuda.is_available():
|
||||
raise SystemExit("CUDA is required for ModelOpt FP8 quantization.")
|
||||
|
||||
mtq = _require_modelopt()
|
||||
model_path, output_dir = _ensure_paths(args)
|
||||
dtype = _select_dtype(args.dtype)
|
||||
prompts = _build_prompts(args)
|
||||
weight_block_size = _parse_block_size(args.weight_block_size)
|
||||
|
||||
print("Quantization plan:")
|
||||
print(f" input: {args.model}")
|
||||
print(f" output: {output_dir}")
|
||||
print(f" dtype: {dtype}")
|
||||
print(f" height/width: {args.height}x{args.width}")
|
||||
print(f" num_frames: {args.num_frames}")
|
||||
print(f" calib_size: {len(prompts)}")
|
||||
print(f" calib_steps: {args.calib_steps}")
|
||||
print(f" quantize_mha: {args.quantize_mha}")
|
||||
print(
|
||||
f" weight strategy: {'block-wise ' + str(weight_block_size) if weight_block_size else 'per-tensor (default)'}"
|
||||
)
|
||||
|
||||
pipe = _load_pipeline(model_path, dtype)
|
||||
transformers = _list_transformers(pipe)
|
||||
if not transformers:
|
||||
raise SystemExit("Pipeline has no transformer or transformer_2 attribute.")
|
||||
print(f" found {len(transformers)} transformer(s): {', '.join(a for a, _ in transformers)}")
|
||||
|
||||
quant_config = copy.deepcopy(mtq.FP8_DEFAULT_CFG)
|
||||
if weight_block_size is not None:
|
||||
quant_config["quant_cfg"]["*weight_quantizer"] = {
|
||||
"num_bits": (4, 3),
|
||||
"block_sizes": {-1: weight_block_size[1], -2: weight_block_size[0]},
|
||||
}
|
||||
print(
|
||||
f" -> overriding weight quantizer with block_sizes={weight_block_size} "
|
||||
f"({weight_block_size[0]}x{weight_block_size[1]} tiles)"
|
||||
)
|
||||
|
||||
forward_loop = _build_forward_loop(pipe, args, prompts)
|
||||
|
||||
# Quantize each DiT. The forward_loop runs the full pipeline, so quantizers
|
||||
# on every attached transformer (A14B: transformer + transformer_2) get
|
||||
# calibrated during the same passes.
|
||||
for attr, backbone in transformers:
|
||||
print(f"\nQuantizing {attr}...")
|
||||
quantized = mtq.quantize(backbone, quant_config, forward_loop)
|
||||
if quantized is not None:
|
||||
setattr(pipe, attr, quantized)
|
||||
_disable_known_problematic_quantizers(mtq, getattr(pipe, attr), quantize_mha=args.quantize_mha)
|
||||
|
||||
# Export weights to FP8 only AFTER all transformers are calibrated. Exporting
|
||||
# inside the loop would convert transformer's weights to real FP8, and the
|
||||
# next transformer's forward_loop (full pipeline) would then fake-quantize an
|
||||
# already-FP8 weight -> "fake_e4m3fy not implemented for Float8_e4m3fn".
|
||||
print("\nForcing FP8 weight serialization (Wan2.2 isn't in ModelOpt's recognized-model registry,")
|
||||
print("so we have to call the per-weight export helper ourselves)...")
|
||||
total_exported = 0
|
||||
for attr, backbone in _list_transformers(pipe):
|
||||
exported = _force_export_quantized_weights(backbone, dtype)
|
||||
print(f" -> {exported} weights converted to FP8 in {attr}")
|
||||
total_exported += exported
|
||||
|
||||
if total_exported == 0:
|
||||
raise SystemExit(
|
||||
"No quantized weights were exported. Calibration may have skipped every layer "
|
||||
"(check the disable_quantizer regex) or `mtq.quantize` did not actually wrap any "
|
||||
"weight quantizers."
|
||||
)
|
||||
|
||||
print("\nSaving pipeline with FP8 transformer(s)...")
|
||||
_save_pipeline_with_fp8_transformer(pipe, model_path, output_dir)
|
||||
_patch_quant_config(output_dir, weight_block_size=weight_block_size)
|
||||
print(f"Saved to: {output_dir}")
|
||||
_summarize_export(output_dir)
|
||||
|
||||
print("\nNext: validate the checkpoint with vllm-omni:")
|
||||
print(
|
||||
" python examples/offline_inference/text_to_video/text_to_video.py \\\n"
|
||||
f" --model {output_dir} \\\n"
|
||||
" --quantization fp8 \\\n"
|
||||
" --prompt 'A dog running across a field of golden wheat.' \\\n"
|
||||
f" --height {args.height} --width {args.width} --num-frames {args.num_frames} \\\n"
|
||||
" --num-inference-steps 30 --guidance-scale 5.0 --seed 42 \\\n"
|
||||
" --output outputs/wan22_modelopt_fp8.mp4"
|
||||
)
|
||||
print(
|
||||
"\n (--quantization fp8 is auto-upgraded to ModelOpt FP8 at runtime because the "
|
||||
"checkpoint's config.json has modelopt metadata.)"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -4,6 +4,17 @@ This recipe uses online FP8 weight quantization, tiled VAE decode, and a single
|
||||
resident partition. GB10 is a unified-memory platform, so unlike the discrete-GPU
|
||||
recipes it uses **no offload of any kind** — see the capacity note below.
|
||||
|
||||
Validated on:
|
||||
|
||||
- Host: DGX Spark (GB10), aarch64
|
||||
- GPUs: 1 (unified memory)
|
||||
- Driver: <FILL_DRIVER_VERSION>
|
||||
- vLLM: 0.26.0
|
||||
- vLLM-Omni: `main` at `e1aa6eae75c460cd1893bc320546e81e66973831`
|
||||
- Workloads: T2VA and Ref2VA, 960x576, `duration=8.0`, 50 steps, `flow_shift=12`,
|
||||
`seed=1101`
|
||||
|
||||
|
||||
## Capacity requirements
|
||||
|
||||
| Resource | DGX Spark (GB10) |
|
||||
@@ -11,7 +22,8 @@ recipes it uses **no offload of any kind** — see the capacity note below.
|
||||
| Unified memory | 128 GB LPDDR5X (~121 GiB usable) |
|
||||
| GPUs | 1 |
|
||||
| Checkpoint storage | 135 GiB per partition |
|
||||
| Measured peak allocator high-water (FP8) | 97.7 GiB |
|
||||
| Measured peak allocator high-water, T2VA (FP8) | 97.7 GiB |
|
||||
| Measured peak allocator high-water, Ref2VA (FP8) | 102.8 GiB |
|
||||
|
||||
`FL2VA` and `Ref2VA` are separate 135 GiB checkpoint partitions. Start one server
|
||||
at a time.
|
||||
@@ -29,9 +41,21 @@ math relative to the RTX and datacenter recipes:
|
||||
fit in 121 GiB. Online FP8 quantizes the DiT only (62 GiB to ~31 GiB); the
|
||||
Qwen3-VL text encoder and both VAEs stay BF16.
|
||||
|
||||
With ~97.7 GiB of the pool held by the allocator at peak, there is roughly 23 GiB
|
||||
of headroom for the OS, page cache, and any other process on the box. Treat this
|
||||
platform as single-tenant while a request is in flight.
|
||||
With ~97.7 GiB of the pool held by the allocator at peak, T2VA leaves roughly
|
||||
23 GiB of headroom for the OS, page cache, and any other process on the box.
|
||||
Ref2VA is tighter on both counts. The allocator high-water rises to 102.8 GiB,
|
||||
and the allocator figure understates true pool usage.
|
||||
|
||||
Sampling `/proc/meminfo` at 1 Hz across a 50-step Ref2VA request on a host with
|
||||
`MemTotal: 127600524 kB` (121.7 GiB), `MemAvailable` started at 8.1 GiB, fell to
|
||||
a trough of **7.4 GiB**, and recovered to 8.9 GiB. At the trough the pool held
|
||||
roughly **114.3 GiB** — 11.4 GiB more than the 102.8 GiB the allocator reports,
|
||||
which is the CUDA context, non-PyTorch allocations, and page cache the header
|
||||
does not count.
|
||||
|
||||
Treat this platform as strictly single-tenant for Ref2VA. There is no room for a
|
||||
second process, and adding reference images, reference videos, or a larger output
|
||||
shape will run into the OOM killer.
|
||||
|
||||
## One GB10: 960x576, 8 seconds
|
||||
|
||||
@@ -94,6 +118,52 @@ ffprobe -v error -show_entries \
|
||||
Both requests return `200 OK` with an MP4 body: H.264 video at the requested
|
||||
geometry plus a 32 kHz stereo AAC track.
|
||||
|
||||
## One GB10: Ref2VA, 960x576, 8 seconds
|
||||
|
||||
`FL2VA` and `Ref2VA` are separate 135 GiB partitions and only one fits at a time.
|
||||
Stop the FL2VA server before starting this one.
|
||||
|
||||
```bash
|
||||
CUDA_VISIBLE_DEVICES=0 \
|
||||
FLASHINFER_DISABLE_VERSION_CHECK=1 \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_OMNI_VIDEO_SYNC_TIMEOUT=14400 \
|
||||
vllm serve /path/to/MiniMax-H3/Ref2VA \
|
||||
--omni --trust-remote-code --host 0.0.0.0 --port 8000 \
|
||||
--init-timeout 3600 \
|
||||
--num-gpus 1 --tensor-parallel-size 1 --text-encoder-tp-size 1 \
|
||||
--usp 1 --ring 1 --vae-patch-parallel-size 1 \
|
||||
--vae-parallel-mode tile --vae-use-tiling \
|
||||
--quantization fp8 --enforce-eager \
|
||||
--diffusion-attention-backend CUDNN_ATTN
|
||||
```
|
||||
|
||||
The 7200 s timeout used for T2VA is too short here: a 50-step Ref2VA request
|
||||
takes over an hour on this platform. Use 14400 s.
|
||||
|
||||
Unlike `t2va`, `ref2va` defaults to a 16:9 output ratio, so `aspect_ratio` can be
|
||||
omitted when `width` and `height` are supplied.
|
||||
|
||||
```bash
|
||||
curl --fail-with-body -sS -X POST http://127.0.0.1:8000/v1/videos/sync \
|
||||
-D ref2va-headers.txt \
|
||||
-w 'client_e2e_s=%{time_total}\n' \
|
||||
--max-time 14400 \
|
||||
-F 'input_reference=@/path/to/reference.jpg;type=image/jpeg' \
|
||||
-F 'prompt=A hand rests on a wooden table in warm evening light, then slowly opens and turns toward the camera, with quiet room ambience.' \
|
||||
-F 'width=960' \
|
||||
-F 'height=576' \
|
||||
-F 'fps=24' \
|
||||
-F 'num_inference_steps=50' \
|
||||
-F 'flow_shift=12' \
|
||||
-F 'seed=1101' \
|
||||
-F 'extra_params={"task":"ref2va","duration":8.0,"audio_flow_shift":3.0}' \
|
||||
-o h3-gb10-ref2va.mp4
|
||||
```
|
||||
|
||||
The response is `200 OK` with an MP4 body: H.264 at the requested geometry plus a
|
||||
32 kHz stereo AAC track. `ffprobe` reports 8.032 s for `duration=8.0`.
|
||||
|
||||
## Measured latency
|
||||
|
||||
Single GB10, FP8, eager, cuDNN attention, tiled VAE, one request at a time:
|
||||
@@ -102,32 +172,69 @@ Single GB10, FP8, eager, cuDNN attention, tiled VAE, one request at a time:
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| T2VA, 4 s, smoke shape | 10 | — | — | 8.68 s | — | 0.49 s | 95.9 s | — |
|
||||
| T2VA, 8 s, 960x576 | 50 | 0.25 s | 2088 s | 42.61 s | 70 s | 4.84 s | 2169.4 s (36 min 9 s) | 97.7 GiB |
|
||||
| Ref2VA, 8 s, 960x576, one image (3 runs) | 10 | 8.43-10.94 s | 665.5-748.8 s | 66.55-74.88 s | 67.9-69.8 s | 3.63 s | 770.3-861.1 s | 102.8 GiB |
|
||||
| Ref2VA, 8 s, 960x576, one image | 50 | 9.66 s | 4039.6 s | 80.79 s | 69.4 s | 3.63 s | 4157.0 s (69 min 17 s) | 102.8 GiB |
|
||||
|
||||
How these were measured:
|
||||
|
||||
- Stage times come from `--enable-diffusion-pipeline-profiler`, read from the
|
||||
`X-Stage-Durations` response header of `/v1/videos/sync`.
|
||||
`X-Stage-Durations` response header of `/v1/videos/sync`. The keys are
|
||||
`MiniMaxH3Pipeline.encode_prompt` (text encode),
|
||||
`MiniMaxH3Pipeline.diffuse` (denoise), and `MiniMaxH3Pipeline.decode`
|
||||
(VAE decode).
|
||||
- MP4 mux is not a header field. It is the
|
||||
`Video response encoding (MP4 bytes)` line in the server log.
|
||||
- Peak memory is the `X-Peak-Memory-MB` header, which reports the CUDA allocator
|
||||
high-water mark (`torch.cuda.max_memory_reserved`) for the request. It excludes
|
||||
the CUDA context and non-PyTorch allocations, so true pool usage is somewhat
|
||||
higher. `nvidia-smi --query-gpu=memory.used` is not usable on GB10 — unified
|
||||
memory has no discrete VRAM to report and the field returns `[N/A]`.
|
||||
- Text encode and VAE decode were measured on a separate 10-step run at the same
|
||||
960x576 / 8 s geometry. Both are independent of step count, so they transfer to
|
||||
the 50-step row unchanged.
|
||||
memory has no discrete VRAM to report and the field returns `[N/A]`. See the
|
||||
capacity section for a `/proc/meminfo` cross-check.
|
||||
- For T2VA, text encode and VAE decode were measured on a separate 10-step run at
|
||||
the same 960x576 / 8 s geometry. Both are independent of step count, so they
|
||||
transfer to the 50-step row unchanged. The Ref2VA MP4 mux figure transfers the
|
||||
same way, from the 10-step run at identical output geometry and duration.
|
||||
- Ref2VA peak memory is byte-identical between the 10-step and 50-step runs
|
||||
(`X-Peak-Memory-MB: 105318.000` in both), confirming that peak is set by output
|
||||
geometry and reference input rather than step count.
|
||||
- Stage times do not sum exactly to end-to-end. Queueing, result transfer, and
|
||||
engine bookkeeping sit outside the profiled stages.
|
||||
|
||||
Denoise dominates at 96% of end-to-end, so step count and geometry are the only
|
||||
knobs that materially change the wall clock on this platform.
|
||||
|
||||
Repeated identical 10-step requests varied by up to 23% in denoise time
|
||||
Ref2VA costs 1.92x the wall clock of T2VA at the same geometry and step count:
|
||||
4157.0 s versus 2169.4 s. Text encoding shows the most dramatic ratio — 9.66 s
|
||||
versus 0.25 s, 39x — but at 0.2% of end-to-end it is not what makes Ref2VA slow.
|
||||
The cost is in denoise. A single reference image expands the Qwen3-VL
|
||||
presentation to 6883 tokens (logged as `MiniMax H3 ref2va Qwen presentation:
|
||||
6883 tokens, 1 reference images`), and those tokens sit in the attention context
|
||||
of every denoise step. Per-step time rises from 42.61 s to 80.79 s, and that
|
||||
1.90x multiplies across all 50 steps. VAE decode is unchanged at ~69 s, as
|
||||
expected for an identical output shape.
|
||||
|
||||
Repeated identical 10-step T2VA requests varied by up to 23% in denoise time
|
||||
(397 s to 490 s) on a machine that had already been under sustained load. Treat
|
||||
these as single-run measurements on one machine, not a throughput guarantee, and
|
||||
expect the slower end of that range for back-to-back jobs.
|
||||
|
||||
Ref2VA per-step time is load-dependent in the same way. On an idle machine the
|
||||
first 10-step request ran at 66.55 s per step. The 50-step request that followed
|
||||
occupied the box for 69 consecutive minutes and averaged 80.79 s per step. Two
|
||||
further 10-step requests taken immediately afterwards settled at 74.88 s and
|
||||
74.31 s — 12.5% slower than the idle run, and within 0.8% of each other, so this
|
||||
is a stable thermal state rather than measurement noise.
|
||||
|
||||
Plan for 81 s per step when sizing a full 50-step Ref2VA job; 66 s per step is
|
||||
only reachable on a cold machine and does not hold for the length of one request.
|
||||
|
||||
Ref2VA text encode converged downward across the four runs (10.94 s, 9.66 s,
|
||||
8.52 s, 8.43 s) as first-touch costs fell away; treat ~8.5 s as steady state.
|
||||
VAE decode was stable at 67.9-69.8 s throughout.
|
||||
|
||||
Set `VLLM_OMNI_VIDEO_SYNC_TIMEOUT` well above the expected request time — the
|
||||
default 1800 s is shorter than a 50-step run on this platform.
|
||||
default 1800 s is shorter than a 50-step run on this platform, and 7200 s is
|
||||
shorter than a 50-step Ref2VA run.
|
||||
|
||||
## Known limitations
|
||||
|
||||
@@ -139,11 +246,17 @@ default 1800 s is shorter than a 50-step run on this platform.
|
||||
- `t2va` rejects requests without an explicit `aspect_ratio`. When `width` and
|
||||
`height` are both supplied they take precedence and `aspect_ratio` only has to
|
||||
name one of the supported ratios.
|
||||
- Sustained load degrades throughput. See the variance note above.
|
||||
- Sustained load degrades throughput. See the variance notes above.
|
||||
- Only one partition is resident. For Ref2VA, stop the FL2VA server and restart
|
||||
the same command with `/path/to/MiniMax-H3/Ref2VA`. Ref2VA reference video
|
||||
count and prompt length increase activation memory; begin with one request at
|
||||
a time and keep the short edge at 576.
|
||||
with `/path/to/MiniMax-H3/Ref2VA`. Ref2VA with one reference image is measured
|
||||
above; reference videos and additional images push the token count higher and
|
||||
were not measured. With `MemAvailable` bottoming out at 7.4 GiB during a
|
||||
single-image request, there is very little room to grow — re-measure before
|
||||
adding references or raising the output shape.
|
||||
- Ref2VA requires a vLLM-Omni build newer than `v0.26.0`. See the version note
|
||||
under **Validated on**: FP8 weight loading and image-only Ref2VA are both
|
||||
broken on `release/v0.26.0`.suggestion is `v0.26.1`.
|
||||
- Online FP8 is incompatible with layerwise offload — the offload path produces a
|
||||
weight stride the Cutlass FP8 kernel rejects. This is not a practical
|
||||
restriction here since offload is unusable on GB10 anyway.
|
||||
```
|
||||
@@ -19,6 +19,11 @@ add files for this repo. Use one Markdown file per model family by default.
|
||||
> Individual pages resolve at `https://recipes.vllm.ai/<provider>/<model-id>`,
|
||||
> e.g. [`bosonai/higgs-audio-v3-tts-4b`](https://recipes.vllm.ai/bosonai/higgs-audio-v3-tts-4b).
|
||||
|
||||
The `Recipe` column in the Supported Models documentation links to the
|
||||
published recipe when available and otherwise to the corresponding recipe in
|
||||
this directory. The recipe files remain the source of truth for commands and
|
||||
validation details.
|
||||
|
||||
Example layout:
|
||||
|
||||
```text
|
||||
|
||||
@@ -69,6 +69,7 @@ def test_h3_fused_rope_matches_reference_and_preserves_unrotated_dims():
|
||||
|
||||
attention = object.__new__(MiniMaxH3Attention)
|
||||
nn.Module.__init__(attention)
|
||||
attention.rot_dim = 96
|
||||
attention.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)
|
||||
attention.rope._forward_method = attention.rope.forward_native
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -43,3 +44,11 @@ def test_model_display_names_require_string_mapping(tmp_path, monkeypatch):
|
||||
|
||||
with pytest.raises(ValueError, match="MODEL_DISPLAY_NAMES_FILE"):
|
||||
generate_examples.load_model_display_names()
|
||||
|
||||
|
||||
def test_only_general_serving_examples_get_generated_pages():
|
||||
general_example = SimpleNamespace(category="offline_inference", path=Path("text_to_image"))
|
||||
model_example = SimpleNamespace(category="offline_inference", path=Path("qwen3_omni"))
|
||||
|
||||
assert generate_examples.is_general_example(general_example)
|
||||
assert not generate_examples.is_general_example(model_example)
|
||||
@@ -68,6 +68,7 @@ class RMSNorm(CustomOp):
|
||||
def __init__(self, hidden_size: int, eps: float = 1e-6, dtype: torch.dtype = torch.float32) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=dtype))
|
||||
self.hidden_size = hidden_size
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def _forward_fused(self, x: torch.Tensor) -> torch.Tensor:
|
||||
@@ -107,6 +108,19 @@ class RMSNorm(CustomOp):
|
||||
except Exception:
|
||||
return self.forward_native(x)
|
||||
|
||||
def forward_musa(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# Preserve the aten::rms_norm graph so dynamic Inductor can fuse the
|
||||
# H3 Q/K norm with its inline RoPE path on MUSA.
|
||||
return F.rms_norm(
|
||||
x,
|
||||
(self.hidden_size,),
|
||||
self.weight,
|
||||
self.variance_epsilon,
|
||||
)
|
||||
|
||||
def forward_npu(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
|
||||
@@ -193,7 +193,16 @@ class RotaryEmbedding(CustomOp):
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.forward_native(x, cos, sin)
|
||||
# Keep the full-dimension NeoX form inline for MUSA compilation.
|
||||
# Other layouts use the shared native implementation.
|
||||
if self.half_head_dim or self.interleaved:
|
||||
return self.forward_native(x, cos, sin)
|
||||
if cos.dim() == 3:
|
||||
cos, sin = cos[0], sin[0]
|
||||
cos = cos.unsqueeze(-2)
|
||||
sin = sin.unsqueeze(-2)
|
||||
x1, x2 = x.chunk(2, dim=-1)
|
||||
return x * cos + torch.cat((-x2, x1), dim=-1) * sin
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
|
||||
@@ -333,6 +333,7 @@ class MiniMaxH3Attention(nn.Module):
|
||||
)
|
||||
self.num_heads = self.qkv_proj.num_heads
|
||||
self.num_kv_heads = self.qkv_proj.num_kv_heads
|
||||
self.rot_dim = 6 * arch.rope_inv_freq_len
|
||||
self.q_norm = _norm(arch.attention_head_dim, eps=arch.qk_norm_eps)
|
||||
self.k_norm = _norm(arch.attention_head_dim, eps=arch.qk_norm_eps)
|
||||
self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)
|
||||
@@ -365,7 +366,7 @@ class MiniMaxH3Attention(nn.Module):
|
||||
x: [T, heads, head_dim]; freqs: [T, rot_dim]. In the unfused path, cos/sin
|
||||
are cast to the activation dtype before the elementwise math.
|
||||
"""
|
||||
rot_dim = freqs.shape[-1]
|
||||
rot_dim = self.rot_dim
|
||||
x_rot, x_pass = x[..., :rot_dim], x[..., rot_dim:]
|
||||
cos = torch.cos(freqs).to(x.dtype) # [T, rot_dim]
|
||||
sin = torch.sin(freqs).to(x.dtype)
|
||||
|
||||
Reference in new issue
Block a user