diff --git a/docs/mkdocs/hooks/generate_examples.py b/docs/mkdocs/hooks/generate_examples.py
index 6cb601b96..f3f96eb2d 100644
--- a/docs/mkdocs/hooks/generate_examples.py
+++ b/docs/mkdocs/hooks/generate_examples.py
@@ -24,6 +24,16 @@ SERVING_MODE_TITLES = {
"online_serving": "Online serving",
}
+# Generated example pages keep their source-derived URLs even when their
+# navigation category changes. Quantization helpers are feature documentation,
+# not user-serving examples, so keep them under the existing Features /
+# Quantization section instead of creating a second top-level Quantization
+# category under Examples.
+EXAMPLE_NAV_TARGETS = {
+ "quantization": "quantization",
+}
+GENERATED_QUANTIZATION_EXAMPLES_TITLE = "Generated Examples"
+
def load_model_display_names() -> dict[str, str]:
try:
@@ -340,26 +350,58 @@ def update_nav_file(examples: list[Example]):
examples_by_category[category] = []
examples_by_category[category].append(example)
- # Build the new Examples section - start with preserved items
- examples_section = preserved_items.copy()
-
- # Add examples grouped by category, sorted by category name
- for category in sorted(examples_by_category.keys()):
- category_examples = sorted(examples_by_category[category], key=lambda e: e.path.stem)
- category_items = []
- for example in category_examples:
+ def category_items(category: str) -> list[dict[str, str]]:
+ items = []
+ for example in sorted(examples_by_category[category], key=lambda e: e.path.stem):
doc_path = EXAMPLE_DOC_DIR / example.category / f"{example.path.stem}.md"
rel_path = doc_path.relative_to(ROOT_DIR / "docs")
- category_items.append({example.nav_title: rel_path.as_posix()})
+ items.append({example.nav_title: rel_path.as_posix()})
+ return items
- if category_items:
- # Format category name (e.g., "offline_inference" -> "Offline Inference")
+ # Build the new Examples section - start with preserved items. Categories
+ # not explicitly routed elsewhere retain the historical Examples behavior.
+ examples_section = preserved_items.copy()
+ for category in sorted(examples_by_category.keys()):
+ if EXAMPLE_NAV_TARGETS.get(category, "examples") != "examples":
+ continue
+ items = category_items(category)
+ if items:
category_title = fix_case(category.replace("_", " ").title())
- examples_section.append({category_title: category_items})
+ examples_section.append({category_title: items})
# Update the nav structure
nav_list[user_guide_idx]["User Guide"][examples_idx]["Examples"] = examples_section
+ features_content = next(
+ (item["Features"] for item in user_guide_content if isinstance(item, dict) and "Features" in item),
+ None,
+ )
+ quantization_content = None
+ if isinstance(features_content, list):
+ quantization_content = next(
+ (item["Quantization"] for item in features_content if isinstance(item, dict) and "Quantization" in item),
+ None,
+ )
+
+ if isinstance(quantization_content, list):
+ # Remove only the group owned by this hook so repeated builds are
+ # idempotent and hand-authored Quantization navigation is retained.
+ quantization_content[:] = [
+ item
+ for item in quantization_content
+ if not (isinstance(item, dict) and GENERATED_QUANTIZATION_EXAMPLES_TITLE in item)
+ ]
+
+ quantization_examples = examples_by_category.get("quantization")
+ if quantization_examples:
+ if not isinstance(quantization_content, list):
+ logger.warning(
+ "Could not find 'User Guide' -> 'Features' -> 'Quantization' section; "
+ "omitting generated quantization examples from navigation"
+ )
+ else:
+ quantization_content.append({GENERATED_QUANTIZATION_EXAMPLES_TITLE: category_items("quantization")})
+
# Write back to file
nav_data["nav"] = nav_list
with open(NAV_FILE, "w", encoding="utf-8") as f:
diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index 585a22bdc..055ba0925 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -17,84 +17,93 @@ th {
}
-| Architecture | Models | Example HF Models | NVIDIA GPU | AMD GPU | Ascend NPU | Intel GPU |
-|--------------|--------|-------------------|------------|---------|-----|-----------|
-| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen1) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | |
-| `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ |
-| `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | |
-| `Gr00tN1d7Pipeline` | GR00T N1.7 | `nvidia/GR00T-N1.7-3B` | ✅︎ | | | |
-| `HunyuanImage3ForCausalMM` | HunyuanImage3.0 (DiT-only) | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2511 | `Qwen/Qwen-Image-Edit-2511` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `GlmImagePipeline` | GLM-Image | `zai-org/GLM-Image` | ✅︎ | ✅︎ | | |
-| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | |
-| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | |
-| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
-| `LTX2DistilledPipeline` | LTX-2 distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
-| `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | |
-| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | | | |
-| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
-| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
-| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ |
-| `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit` | ✅︎ | | | |
-| `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ |
-| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ |
-| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | |
-| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | |
-| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | |
-| `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | |
-| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | |
-| `FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | ✅︎ | ✅︎ | | ✅︎ |
-| `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ |
-| `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ |
-| `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ |
-| `SoulXSingerPipeline` | SoulX-Singer (SVS) | `Soul-AILab/SoulX-Singer` | ✅︎ | | | |
-| `SoulXSingerSVCPipeline` | SoulX-Singer-SVC | `Soul-AILab/SoulX-Singer` (`model-svc.pt`) | ✅︎ | | | |
-| `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | |
-| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `MingTTSForConditionalGeneration` | Ming-omni-tts dense 0.5B | `inclusionAI/Ming-omni-tts-0.5B` | ✅︎ | | | |
-| `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | |
-| `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | | |
-| `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | |
-| `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | |
-| `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | |
-| `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | |
-| `IndexTTS2TalkerForConditionalGeneration` | IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✅︎ | | | |
-| `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
-| `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
-| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |
-| `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | |
-| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ |
-| `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | |
-| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | |
-| `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | |
-| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | |
-| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
-| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
-| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
-|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
-| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | |
-| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-|`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | |
+| Architecture | Models | Example HF Models | NVIDIA GPU | AMD GPU | Ascend NPU | Intel GPU | Recipe |
+|--------------|--------|-------------------|------------|---------|-----|-----------|--------|
+| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen3-Omni.md) |
+| `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen1) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | | — |
+| `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | | — |
+| `Gr00tN1d7Pipeline` | GR00T N1.7 | `nvidia/GR00T-N1.7-3B` | ✅︎ | | | | — |
+| `HunyuanImage3ForCausalMM` | HunyuanImage3.0 (DiT-only) | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | | | | [Published](https://recipes.vllm.ai/Qwen/Qwen-Image) |
+| `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image-2512.md) |
+| `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2511 | `Qwen/Qwen-Image-Edit-2511` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `GlmImagePipeline` | GLM-Image | `zai-org/GLM-Image` | ✅︎ | ✅︎ | | | — |
+| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | | — |
+| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | | — |
+| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | — |
+| `LTX2DistilledPipeline` | LTX-2 distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | | — |
+| `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | — |
+| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | ✅︎ | | | [Published](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) |
+| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | — |
+| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | | — |
+| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit` | ✅︎ | | | | — |
+| `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | | — |
+| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | | — |
+| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
+| `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
+| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | | — |
+| `FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `SoulXSingerPipeline` | SoulX-Singer (SVS) | `Soul-AILab/SoulX-Singer` | ✅︎ | | | | — |
+| `SoulXSingerSVCPipeline` | SoulX-Singer-SVC | `Soul-AILab/SoulX-Singer` (`model-svc.pt`) | ✅︎ | | | | — |
+| `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | | — |
+| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | | | | [Published](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) |
+| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+| `MingTTSForConditionalGeneration` | Ming-omni-tts dense 0.5B | `inclusionAI/Ming-omni-tts-0.5B` | ✅︎ | | | | — |
+| `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | | — |
+| `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | | | — |
+| `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | | — |
+| `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | | — |
+| `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | | — |
+| `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | | [Published](https://recipes.vllm.ai/bosonai/higgs-audio-v3-tts-4b) |
+| `IndexTTS2TalkerForConditionalGeneration` | IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✅︎ | | | | — |
+| `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | | — |
+| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | | — |
+| `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | | — |
+| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ | — |
+| `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | | — |
+| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | | — |
+| `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | | — |
+| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | | — |
+| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | | — |
+| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
+| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
+|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | | — |
+| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | — |
+| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
+|`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
+
+The `Recipe` column links the corresponding deployment recipe. `Published` is
+preferred when a corresponding `recipes.vllm.ai` page exists; otherwise the
+link points to the repository recipe. `—` means that no recipe link has been
+added for that model row yet.
+
+For rows with an audited recipe link, the hardware checkmarks reflect hardware
+explicitly documented in that recipe. Other rows retain the implementation
+support metadata until their recipe is audited.
✅︎ indicates the model is supported on that backend. Empty cells mean not listed as supported on that backend.
diff --git a/examples/quantization/check_modelopt_fp8_export.py b/examples/quantization/check_modelopt_fp8_export.py
deleted file mode 100644
index 90d952a8f..000000000
--- a/examples/quantization/check_modelopt_fp8_export.py
+++ /dev/null
@@ -1,247 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Verify a ModelOpt FP8 diffusers checkpoint exported by
-quantize_wan2_2_modelopt_fp8.py / quantize_hunyuanvideo_15_modelopt_fp8.py.
-
-Three checks:
- A. transformer/config.json has a sane quantization_config block.
- B. transformer/*.safetensors contains FP8 (float8_e4m3fn) quantized tensors.
- C. transformer disk size is materially smaller than a BF16 baseline.
-
-Example:
- python examples/quantization/check_modelopt_fp8_export.py \\
- --output ./hv15-480p-modelopt-fp8
-
- # Optional: compare disk size against a local or HF BF16 baseline.
- python examples/quantization/check_modelopt_fp8_export.py \\
- --output ./hv15-480p-modelopt-fp8 \\
- --baseline hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from collections import Counter
-from pathlib import Path
-
-SUPPORTED_ALGOS = ("FP8",)
-
-
-def _check_config(transformer_dir: Path) -> tuple[int, str | None]:
- """Returns (status, quant_algo). status: 0 pass, 1 fail, 2 warn."""
- cfg_path = transformer_dir / "config.json"
- if not cfg_path.exists():
- print(f"[FAIL] {cfg_path} missing.")
- return 1, None
-
- with cfg_path.open(encoding="utf-8") as f:
- cfg = json.load(f)
-
- qc = cfg.get("quantization_config")
- if not isinstance(qc, dict):
- print(f"[FAIL] No `quantization_config` block in {cfg_path}.")
- return 1, None
-
- print(f"[A] quantization_config from {cfg_path}:")
- print(json.dumps(qc, indent=2))
-
- quant_algo = qc.get("quant_algo")
- issues = []
- if qc.get("quant_method") != "modelopt":
- issues.append(f"quant_method={qc.get('quant_method')!r} (expected 'modelopt')")
- if quant_algo not in SUPPORTED_ALGOS:
- issues.append(
- f"quant_algo={quant_algo!r} (expected one of {SUPPORTED_ALGOS} — vllm-omni adapter may not auto-detect)"
- )
-
- if issues:
- print("[A] WARN — config looks incomplete:")
- for issue in issues:
- print(f" - {issue}")
- return 2, quant_algo
- print(f"[A] PASS — config looks correct (quant_algo={quant_algo}).")
- return 0, quant_algo
-
-
-def _read_safetensors_header(path: Path) -> dict:
- """Read the JSON header of a safetensors file. Bypass-safe — doesn't materialize tensors.
-
- Returns {tensor_name: {'dtype': 'F8_E4M3', 'shape': [...], 'data_offsets': [...]}}.
- Header dtype strings: F8_E4M3, F8_E5M2, BF16, F16, F32, F64, I8, I16, I32, I64, BOOL, U8, ...
- """
- import struct
-
- with open(path, "rb") as f:
- header_len = struct.unpack(" str:
- """Infer per-tensor vs per-channel vs per-block from sample weight_scale shapes.
-
- ModelOpt block-wise produces shapes like `[16, 1, 16, 1]` (broadcasting dims of 1
- interleaved with block-count dims). We count "meaningful" dims — ones with size > 1 —
- and classify: 0 meaningful dims = per-tensor (scalar), 1 = per-channel, 2+ = per-block.
- """
- if not weight_scale_shapes:
- return "no weight_scale tensors found"
-
- def meaningful_dims(shape: list[int]) -> int:
- return sum(1 for d in shape if d > 1)
-
- per_tensor = sum(1 for s in weight_scale_shapes if meaningful_dims(s) == 0)
- per_channel = sum(1 for s in weight_scale_shapes if meaningful_dims(s) == 1)
- per_block = sum(1 for s in weight_scale_shapes if meaningful_dims(s) >= 2)
- total = len(weight_scale_shapes)
- if per_tensor == total:
- return "per-tensor (all scalar scales)"
- if per_channel == total:
- return "per-channel (1 meaningful dim)"
- if per_block == total:
- return "per-block (2+ meaningful dims — e.g. [M//bm, 1, N//bn, 1] for tiles)"
- return f"mixed: per-tensor={per_tensor}, per-channel={per_channel}, per-block={per_block} of {total}"
-
-
-def _check_safetensors(transformer_dir: Path) -> int:
- """Returns 0 on pass, 1 on fail. Reads on-disk dtype from the safetensors header."""
- files = sorted(transformer_dir.glob("*.safetensors"))
- if not files:
- print(f"[FAIL] No *.safetensors in {transformer_dir}.")
- return 1
-
- header_dtype_counts: Counter[str] = Counter()
- sample_quant_weight_keys: list[str] = []
- sample_scale_keys: list[str] = []
- weight_scale_shapes: list[list[int]] = []
- sample_weight_scale_entries: list[tuple[str, list[int]]] = []
-
- for f in files:
- try:
- header = _read_safetensors_header(f)
- except Exception as exc:
- print(f"[B] WARN — could not parse header of {f}: {exc}")
- continue
- for k, info in header.items():
- dtype = info.get("dtype", "?")
- header_dtype_counts[dtype] += 1
- # FP8 stores weights as F8_E4M3 directly.
- if dtype.startswith("F8") and k.endswith(".weight") and len(sample_quant_weight_keys) < 5:
- sample_quant_weight_keys.append(k)
- if k.endswith(("_scale", ".weight_scale", ".input_scale", "_scale_inv")) and len(sample_scale_keys) < 5:
- sample_scale_keys.append(k)
- if k.endswith(".weight_scale"):
- weight_scale_shapes.append(info.get("shape", []))
- if len(sample_weight_scale_entries) < 5:
- sample_weight_scale_entries.append((k, info.get("shape", [])))
-
- print(f"\n[B] On-disk dtype counts across {len(files)} safetensors file(s) (from header, not get_tensor):")
- for dtype, count in sorted(header_dtype_counts.items(), key=lambda kv: -kv[1]):
- marker = " <-- FP8" if dtype.startswith("F8") else ""
- print(f" {dtype:10s} {count:>6d}{marker}")
-
- quant_count = sum(c for d, c in header_dtype_counts.items() if d.startswith("F8"))
- if quant_count == 0:
- print("[B] FAIL — no FP8 tensors on disk. Calibration likely did not actually quantize the weights.")
- return 1
-
- print(f"[B] PASS — {quant_count} FP8 tensors stored on disk.")
- if sample_quant_weight_keys:
- print(f" sample quantized weight tensors: {sample_quant_weight_keys[:3]}")
- if sample_scale_keys:
- print(f" sample scale tensors: {sample_scale_keys[:3]}")
- print(" (Note: torch's get_tensor() may return these as bf16 views on some versions —")
- print(" irrelevant; vLLM's loader uses native FP8 ops.)")
-
- # Weight-scale granularity — per-tensor (scalar) vs per-channel (1-D) vs per-block (N-D).
- print(f"\n weight_scale granularity: {_classify_weight_scale_granularity(weight_scale_shapes)}")
- for key, shape in sample_weight_scale_entries[:3]:
- print(f" {key}: shape {shape}")
- return 0
-
-
-def _disk_size_gib(p: Path) -> float:
- return sum(f.stat().st_size for f in p.rglob("*") if f.is_file()) / (1024**3)
-
-
-def _check_size_vs_baseline(transformer_dir: Path, baseline: str | None) -> int:
- """Returns 0 always (informational only)."""
- quant_size = _disk_size_gib(transformer_dir)
- print(f"\n[C] FP8 transformer disk size: {quant_size:.2f} GiB")
-
- if baseline is None:
- print("[C] SKIP — pass --baseline to compare against BF16.")
- return 0
-
- baseline_path = Path(baseline)
- if not baseline_path.exists():
- # Try HF download.
- try:
- from huggingface_hub import snapshot_download
- except ImportError:
- print("[C] SKIP — huggingface_hub not installed and baseline not a local path.")
- return 0
- print(f" Downloading baseline transformer from HF: {baseline}")
- baseline_path = Path(snapshot_download(baseline, allow_patterns=["transformer/*"]))
-
- bf16_dir = baseline_path / "transformer" if (baseline_path / "transformer").exists() else baseline_path
- bf16_size = _disk_size_gib(bf16_dir)
- if bf16_size == 0:
- print(f"[C] WARN — baseline transformer dir empty: {bf16_dir}")
- return 0
-
- # Expected reduction for FP8: ~50%.
- min_reduction = 30
- reduction = (1 - quant_size / bf16_size) * 100
- print(f"[C] BF16 baseline transformer disk size: {bf16_size:.2f} GiB ({bf16_dir})")
- print(f"[C] Disk reduction: {reduction:.1f}% (FP8 transformer is {quant_size / bf16_size:.0%} of BF16)")
- if reduction < min_reduction:
- print(
- f"[C] WARN — FP8 should typically reduce disk by ~40-50%; <{min_reduction}% suggests partial quantization."
- )
- return 0
-
-
-def main() -> None:
- p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
- p.add_argument("--output", required=True, help="Path to the exported ModelOpt FP8 checkpoint root.")
- p.add_argument(
- "--baseline",
- default=None,
- help="Optional BF16 baseline (local diffusers dir or HF id) for disk-size comparison.",
- )
- args = p.parse_args()
-
- out_root = Path(args.output).expanduser().resolve()
- transformer_dir = out_root / "transformer"
- if not transformer_dir.exists():
- print(f"[FAIL] {transformer_dir} does not exist.")
- sys.exit(1)
-
- print(f"Checking: {out_root}\n")
-
- fail = 0
- config_status, _quant_algo = _check_config(transformer_dir)
- fail |= config_status
- fail |= _check_safetensors(transformer_dir)
- _check_size_vs_baseline(transformer_dir, args.baseline)
-
- print()
- if fail == 0:
- print("=" * 60)
- print("ALL CHECKS PASSED — checkpoint looks ready for vllm-omni serving.")
- elif fail == 1:
- print("=" * 60)
- print("FAILURES detected — calibration may need to be re-run.")
- sys.exit(1)
- else:
- print("=" * 60)
- print("WARNINGS only — checkpoint may serve but with caveats. See [A] above.")
-
-
-if __name__ == "__main__":
- main()
diff --git a/examples/quantization/quantize_hunyuanvideo_15_modelopt_fp8.py b/examples/quantization/quantize_hunyuanvideo_15_modelopt_fp8.py
deleted file mode 100644
index e6e3dd5c5..000000000
--- a/examples/quantization/quantize_hunyuanvideo_15_modelopt_fp8.py
+++ /dev/null
@@ -1,448 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Quantize HunyuanVideo-1.5 (480p T2V) to a ModelOpt FP8 Hugging Face checkpoint.
-
-Calibrates the DiT transformer using a small video prompt set and exports a
-diffusers-style directory whose transformer carries ModelOpt FP8 metadata.
-The exported checkpoint is consumable by vllm-omni's ModelOpt FP8 adapter
-(see vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py).
-
-Layers kept full precision match the #2728 / #2795 pattern: modulation,
-AdaLayerNorm, entry/exit projections, embeddings, the token refiner path,
-and final proj_out. MHA quantizers are off by default; HV-1.5 self-attention
-empirically degrades under FP8 (see #2920 ablation).
-
-Example:
- python examples/quantization/quantize_hunyuanvideo_15_modelopt_fp8.py \\
- --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \\
- --output ./hv15-480p-modelopt-fp8 \\
- --overwrite
-"""
-
-from __future__ import annotations
-
-import argparse
-import copy
-import json
-import re
-import shutil
-import sys
-from pathlib import Path
-from typing import Any
-
-import torch
-from diffusers import DiffusionPipeline
-
-DEFAULT_PROMPTS = [
- "A dog running across a field of golden wheat.",
- "An astronaut riding a horse across the surface of Mars, red dust swirling, cinematic wide shot.",
- "A hummingbird hovering in front of a vibrant red flower, slow motion, macro shot.",
- "A crackling campfire at night under a starry sky, sparks rising into the dark.",
- "An underwater shot of a coral reef with tropical fish swimming by, sun rays piercing the water.",
- "A close-up of a blooming rose covered in morning dew, soft natural light.",
- "A peaceful mountain village at dawn, mist rolling over the rooftops, cinematic establishing shot.",
- "A skateboarder doing a kickflip in an urban plaza, slow motion, golden hour lighting.",
-]
-
-
-def _build_parser() -> argparse.ArgumentParser:
- p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
- p.add_argument("--model", required=True, help="Input HV-1.5 diffusers directory or HF id.")
- p.add_argument("--output", required=True, help="Output directory for the ModelOpt FP8 checkpoint.")
- p.add_argument("--dtype", choices=("bfloat16", "float16"), default="bfloat16")
- p.add_argument("--height", type=int, default=480)
- p.add_argument("--width", type=int, default=832)
- p.add_argument(
- "--num-frames",
- type=int,
- default=33,
- help="Frames per calibration sample. 33 matches the typical short benchmark.",
- )
- p.add_argument("--guidance-scale", type=float, default=6.0)
- p.add_argument(
- "--calib-steps",
- type=int,
- default=10,
- help="Denoising steps per calibration prompt (10 is enough for amax statistics).",
- )
- p.add_argument("--calib-size", type=int, default=8, help="How many prompts to use for calibration.")
- p.add_argument("--seed", type=int, default=42)
- p.add_argument(
- "--prompt",
- action="append",
- default=[],
- help="Custom calibration prompt. Repeat to provide multiple.",
- )
- p.add_argument(
- "--quantize-mha",
- action="store_true",
- help="Enable FP8 attention K/V/softmax quantizers. Off by default — empirically degrades HV-1.5 video output.",
- )
- p.add_argument(
- "--weight-block-size",
- type=str,
- default=None,
- help="Per-block weight quantization as 'M,N' (e.g. '128,128' for 128x128 tiles). "
- "Default: per-tensor (one scale per linear). Block-wise typically gives tighter quality at "
- "negligible memory cost. Static FP8 is exempt from upstream vLLM's online block-wise gate.",
- )
- p.add_argument("--overwrite", action="store_true", help="Replace an existing output directory.")
- return p
-
-
-def _parse_block_size(spec: str | None) -> list[int] | None:
- if spec is None:
- return None
- parts = [int(x) for x in spec.split(",") if x.strip()]
- if len(parts) != 2:
- raise SystemExit(f"--weight-block-size must be 'M,N' (2 ints), got {spec!r}")
- return parts
-
-
-def _require_modelopt() -> Any:
- try:
- import modelopt.torch.quantization as mtq
- except ModuleNotFoundError as exc:
- raise SystemExit(
- "NVIDIA ModelOpt is not installed. Install with:\n"
- " pip install 'nvidia-modelopt[all]'\n"
- f"Original error: {exc}"
- ) from exc
- return mtq
-
-
-def _ensure_paths(args: argparse.Namespace) -> tuple[str, Path]:
- model_path = args.model
- output_dir = Path(args.output).expanduser().resolve()
- if output_dir.exists():
- if not args.overwrite:
- raise SystemExit(f"Output directory already exists: {output_dir}\nPass --overwrite to replace it.")
- shutil.rmtree(output_dir)
- return model_path, output_dir
-
-
-def _select_dtype(name: str) -> torch.dtype:
- return {"bfloat16": torch.bfloat16, "float16": torch.float16}[name]
-
-
-def _build_prompts(args: argparse.Namespace) -> list[str]:
- prompts = args.prompt or DEFAULT_PROMPTS
- if args.calib_size <= 0:
- raise SystemExit("--calib-size must be positive.")
- if len(prompts) < args.calib_size:
- repeats = (args.calib_size + len(prompts) - 1) // len(prompts)
- prompts = (prompts * repeats)[: args.calib_size]
- return prompts[: args.calib_size]
-
-
-# Layers to KEEP at full precision (mirror of the #2920 wiring + #2728/#2795 skip pattern).
-# - x_embedder, image_embedder, context_embedder*, time_embed*, cond_type_embed: entry/embedding
-# - norm_out, norm1*.linear, norm1_context*.linear, norm2*, norm2_context*: AdaLayerNorm modulation
-# - proj_out: final output projection
-# - token_refiner*: text-encoder refinement uses diffusers raw nn.Linear
-def _filter_func_hv15(name: str) -> bool:
- pattern = re.compile(
- r"(proj_out.*|"
- r".*(x_embedder|image_embedder|context_embedder|context_embedder_2|"
- r"time_embed|cond_type_embed|"
- r"norm_out|norm1\.linear|norm1_context\.linear|norm2|norm2_context|"
- r"token_refiner).*)"
- )
- return pattern.match(name) is not None
-
-
-def _mha_filter_func(name: str) -> bool:
- pattern = re.compile(
- r".*(q_bmm_quantizer|k_bmm_quantizer|v_bmm_quantizer|softmax_quantizer|bmm2_output_quantizer).*"
- )
- return pattern.match(name) is not None
-
-
-def _disable_known_problematic_quantizers(mtq: Any, backbone: torch.nn.Module, *, quantize_mha: bool) -> None:
- if not hasattr(mtq, "disable_quantizer"):
- return
- mtq.disable_quantizer(backbone, _filter_func_hv15)
- if not quantize_mha:
- mtq.disable_quantizer(backbone, _mha_filter_func)
-
-
-def _load_pipeline(model_path: str, dtype: torch.dtype) -> DiffusionPipeline:
- pipe = DiffusionPipeline.from_pretrained(model_path, torch_dtype=dtype)
- if hasattr(pipe, "set_progress_bar_config"):
- pipe.set_progress_bar_config(disable=True)
- pipe.to("cuda")
- return pipe
-
-
-def _build_forward_loop(pipe: DiffusionPipeline, args: argparse.Namespace, prompts: list[str]):
- generator = torch.Generator(device="cuda")
-
- # Try to set guidance on the pipeline's guider object up front (modern
- # diffusers HV-1.5 uses a Guider abstraction, not a per-call kwarg). Falls
- # back silently — calibration uses whatever default the pipeline ships with.
- guider = getattr(pipe, "guider", None)
- if guider is not None and hasattr(guider, "guidance_scale"):
- try:
- guider.guidance_scale = args.guidance_scale
- except Exception:
- pass
-
- base_kwargs = dict(
- height=args.height,
- width=args.width,
- num_frames=args.num_frames,
- num_inference_steps=args.calib_steps,
- output_type="latent",
- )
-
- def forward_loop(*_unused_args, **_unused_kwargs) -> None:
- with torch.inference_mode():
- for idx, prompt in enumerate(prompts):
- generator.manual_seed(args.seed + idx)
- # Try with guidance_scale first; fall back without on TypeError
- # for pipelines (like HV-1.5) that take CFG via guider config.
- try:
- pipe(prompt=prompt, generator=generator, guidance_scale=args.guidance_scale, **base_kwargs)
- except TypeError as exc:
- if "guidance_scale" not in str(exc):
- raise
- pipe(prompt=prompt, generator=generator, **base_kwargs)
-
- return forward_loop
-
-
-def _summarize_export(output_dir: Path) -> None:
- cfg_path = output_dir / "transformer" / "config.json"
- if not cfg_path.exists():
- print(f"[warn] {cfg_path} missing.", file=sys.stderr)
- return
- with cfg_path.open(encoding="utf-8") as f:
- cfg = json.load(f)
- qc = cfg.get("quantization_config")
- if not isinstance(qc, dict):
- print("[warn] No quantization_config in transformer/config.json.", file=sys.stderr)
- return
- print("Export summary:")
- print(f" quant_method: {qc.get('quant_method')}")
- print(f" quant_algo: {qc.get('quant_algo')}")
- producer = qc.get("producer")
- if isinstance(producer, dict):
- print(f" producer: {producer.get('name')} {producer.get('version')}")
- print(f" config path: {cfg_path}")
-
-
-def _force_export_quantized_weights(backbone: torch.nn.Module, dtype: torch.dtype) -> int:
- """Convert in-memory weights of quantized modules to actual FP8 storage.
-
- `export_hf_checkpoint` skips this step for unknown model types (HV-1.5 isn't
- in ModelOpt's recognized-model registry), so we must call the per-weight
- export helper ourselves. Same workaround as the HunyuanImage-3 calibration
- helper.
- """
- from modelopt.torch.export.quant_utils import (
- QUANTIZATION_NONE,
- get_quantization_format,
- quantizer_attr_names,
- weight_attr_names,
- )
- from modelopt.torch.export.unified_export_hf import _export_quantized_weight
-
- exported = 0
- for name, module in backbone.named_modules():
- try:
- quantization_format = get_quantization_format(module)
- except Exception as exc:
- print(f"[warn] Could not inspect quantization format for {name}: {exc}", file=sys.stderr)
- continue
- if quantization_format == QUANTIZATION_NONE:
- continue
- for weight_name in weight_attr_names(module):
- quantizer_attrs = quantizer_attr_names(weight_name)
- weight_quantizer = getattr(module, quantizer_attrs.weight_quantizer, None)
- if weight_quantizer is None or not getattr(weight_quantizer, "is_enabled", False):
- continue
- _export_quantized_weight(module, dtype, weight_name)
- exported += 1
- return exported
-
-
-def _hv15_quant_config_block(weight_block_size: list[int] | None = None) -> dict:
- """Mirror ModelOpt FP8 metadata expected by vllm-omni's adapter (#2913).
-
- Same shape as the HunyuanImage-3 author's _hunyuan_quant_config(). When
- `weight_block_size` is given, advertise block-wise weight quantization in
- the saved metadata (so consumers know to expect multi-element scale tensors).
- """
- weights_cfg: dict = {"dynamic": False, "num_bits": 8, "type": "float"}
- if weight_block_size is not None:
- weights_cfg["strategy"] = "block"
- weights_cfg["block_structure"] = f"{weight_block_size[0]}x{weight_block_size[1]}"
- return {
- "config_groups": {
- "group_0": {
- "input_activations": {"dynamic": False, "num_bits": 8, "type": "float"},
- "weights": weights_cfg,
- "targets": ["Linear"],
- }
- },
- "ignore": [
- "context_embedder*",
- "context_embedder_2*",
- "cond_type_embed*",
- "image_embedder*",
- "norm1.linear*",
- "norm1_context.linear*",
- "norm2*",
- "norm2_context*",
- "norm_out*",
- "proj_out*",
- "time_embed*",
- "token_refiner*",
- "x_embedder*",
- ],
- "producer": {"name": "modelopt"},
- "quant_algo": "FP8",
- "quant_method": "modelopt",
- }
-
-
-def _patch_quant_config(output_dir: Path, weight_block_size: list[int] | None = None) -> None:
- """Inject quant_algo: FP8 + config_groups into transformer/config.json so
- vllm-omni's adapter (#2913) recognises the checkpoint as ModelOpt FP8."""
- cfg_path = output_dir / "transformer" / "config.json"
- with cfg_path.open(encoding="utf-8") as f:
- cfg = json.load(f)
-
- new_qc = _hv15_quant_config_block(weight_block_size=weight_block_size)
- existing = cfg.get("quantization_config")
- if isinstance(existing, dict):
- producer = existing.get("producer")
- if isinstance(producer, dict):
- new_qc["producer"] = producer
-
- cfg["quantization_config"] = new_qc
- with cfg_path.open("w", encoding="utf-8") as f:
- json.dump(cfg, f, indent=2)
-
-
-def _save_pipeline_with_fp8_transformer(
- pipe: DiffusionPipeline,
- model_path: str,
- output_dir: Path,
- max_shard_size: str = "5GB",
-) -> None:
- """Save the pipeline with the (now FP8) transformer.
-
- Copies the source directory verbatim except for `transformer/`, then
- saves the transformer with quantizers hidden so the state dict contains
- only the FP8 weights + scale tensors.
- """
- from modelopt.torch.export.diffusers_utils import hide_quantizers_from_state_dict
-
- src = Path(model_path)
- if not src.exists():
- from huggingface_hub import snapshot_download
-
- src = Path(snapshot_download(model_path))
-
- if output_dir.exists():
- shutil.rmtree(output_dir)
- shutil.copytree(src, output_dir, ignore=shutil.ignore_patterns("transformer"))
-
- transformer_out = output_dir / "transformer"
- # `hide_quantizers_from_state_dict` walks named_modules(); pass the actual
- # nn.Module (transformer), not the diffusers Pipeline wrapper.
- with hide_quantizers_from_state_dict(pipe.transformer):
- pipe.transformer.save_pretrained(
- str(transformer_out),
- safe_serialization=True,
- max_shard_size=max_shard_size,
- )
-
-
-def main() -> None:
- args = _build_parser().parse_args()
- if not torch.cuda.is_available():
- raise SystemExit("CUDA is required for ModelOpt FP8 quantization.")
-
- mtq = _require_modelopt()
- model_path, output_dir = _ensure_paths(args)
- dtype = _select_dtype(args.dtype)
- prompts = _build_prompts(args)
-
- print("Quantization plan:")
- weight_block_size = _parse_block_size(args.weight_block_size)
-
- print(f" input: {args.model}")
- print(f" output: {output_dir}")
- print(f" dtype: {dtype}")
- print(f" height/width: {args.height}x{args.width}")
- print(f" num_frames: {args.num_frames}")
- print(f" calib_size: {len(prompts)}")
- print(f" calib_steps: {args.calib_steps}")
- print(f" quantize_mha: {args.quantize_mha}")
- print(
- f" weight strategy: {'block-wise ' + str(weight_block_size) if weight_block_size else 'per-tensor (default)'}"
- )
-
- pipe = _load_pipeline(model_path, dtype)
- backbone = pipe.transformer
-
- quant_config = copy.deepcopy(mtq.FP8_DEFAULT_CFG)
- if weight_block_size is not None:
- # Switch from per-tensor (default) to block-wise weight quantization.
- # ModelOpt's wildcard "*weight_quantizer" matches every linear's weight quantizer.
- quant_config["quant_cfg"]["*weight_quantizer"] = {
- "num_bits": (4, 3), # E4M3 (FP8 weights, same as default)
- "block_sizes": {-1: weight_block_size[1], -2: weight_block_size[0]},
- }
- print(
- f" -> overriding weight quantizer with block_sizes={weight_block_size} "
- f"({weight_block_size[0]}x{weight_block_size[1]} tiles)"
- )
-
- forward_loop = _build_forward_loop(pipe, args, prompts)
- quantized = mtq.quantize(backbone, quant_config, forward_loop)
- if quantized is not None:
- pipe.transformer = quantized
- backbone = quantized
-
- _disable_known_problematic_quantizers(mtq, backbone, quantize_mha=args.quantize_mha)
-
- print("\nForcing FP8 weight serialization (HV-1.5 isn't in ModelOpt's recognized-model registry,")
- print("so we have to call the per-weight export helper ourselves)...")
- exported = _force_export_quantized_weights(backbone, dtype)
- print(f" -> {exported} weights converted to FP8 in memory")
- if exported == 0:
- raise SystemExit(
- "No quantized weights were exported. Calibration may have skipped every layer "
- "(check the disable_quantizer regex) or `mtq.quantize` did not actually wrap any "
- "weight quantizers."
- )
-
- print("\nSaving pipeline with FP8 transformer...")
- _save_pipeline_with_fp8_transformer(pipe, model_path, output_dir)
- _patch_quant_config(output_dir, weight_block_size=weight_block_size)
- print(f"Saved to: {output_dir}")
- _summarize_export(output_dir)
-
- print("\nNext: validate the checkpoint with vllm-omni:")
- print(
- " python examples/offline_inference/text_to_video/text_to_video.py \\\n"
- f" --model {output_dir} \\\n"
- " --quantization fp8 \\\n"
- " --prompt 'A dog running across a field of golden wheat.' \\\n"
- f" --height {args.height} --width {args.width} --num-frames {args.num_frames} \\\n"
- " --num-inference-steps 30 --guidance-scale 6.0 --seed 42 \\\n"
- " --output outputs/hv15_modelopt_fp8.mp4 \\\n"
- " --enforce-eager"
- )
- print(
- "\n (--quantization fp8 is auto-upgraded to ModelOpt FP8 at runtime because the "
- "checkpoint's config.json has modelopt metadata.)"
- )
-
-
-if __name__ == "__main__":
- main()
diff --git a/examples/quantization/quantize_wan2_2_modelopt_fp8.py b/examples/quantization/quantize_wan2_2_modelopt_fp8.py
deleted file mode 100644
index 6b6cc42a4..000000000
--- a/examples/quantization/quantize_wan2_2_modelopt_fp8.py
+++ /dev/null
@@ -1,471 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Quantize Wan2.2 (TI2V-5B, 704x1280 T2V) to a ModelOpt FP8 Hugging Face checkpoint.
-
-Calibrates the DiT transformer using a small video prompt set and exports a
-diffusers-style directory whose transformer carries ModelOpt FP8 metadata.
-The exported checkpoint is consumable by vllm-omni's ModelOpt FP8 adapter
-(see vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py).
-
-Layers kept full precision match the #2728 / #2795 pattern: condition embedder
-(time/text/image), patch embedding, modulation (scale_shift_table), final
-norm + proj_out, and sequence-parallel helpers. All attention + FFN linears
-are quantized — static calibration handles the numerics that online FP8
-couldn't (see #2920 ablation).
-
-Default target is `Wan-AI/Wan2.2-TI2V-5B-Diffusers`, the dense 5B variant that
-fits 80GB BF16. The A14B MoE variants need 2+ GPUs and are out of scope here.
-
-Example:
- python examples/quantization/quantize_wan2_2_modelopt_fp8.py \\
- --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \\
- --output ./wan22-ti2v-modelopt-fp8 \\
- --overwrite
-"""
-
-from __future__ import annotations
-
-import argparse
-import copy
-import json
-import re
-import shutil
-import sys
-from pathlib import Path
-from typing import Any
-
-import torch
-from diffusers import DiffusionPipeline
-
-DEFAULT_PROMPTS = [
- "A dog running across a field of golden wheat.",
- "An astronaut riding a horse across the surface of Mars, red dust swirling, cinematic wide shot.",
- "A hummingbird hovering in front of a vibrant red flower, slow motion, macro shot.",
- "A crackling campfire at night under a starry sky, sparks rising into the dark.",
- "An underwater shot of a coral reef with tropical fish swimming by, sun rays piercing the water.",
- "A close-up of a blooming rose covered in morning dew, soft natural light.",
- "A peaceful mountain village at dawn, mist rolling over the rooftops, cinematic establishing shot.",
- "A skateboarder doing a kickflip in an urban plaza, slow motion, golden hour lighting.",
-]
-
-
-def _build_parser() -> argparse.ArgumentParser:
- p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
- p.add_argument("--model", required=True, help="Input Wan2.2 diffusers directory or HF id.")
- p.add_argument("--output", required=True, help="Output directory for the ModelOpt FP8 checkpoint.")
- p.add_argument("--dtype", choices=("bfloat16", "float16"), default="bfloat16")
- p.add_argument("--height", type=int, default=704, help="Calibration video height (Wan2.2 TI2V-5B native: 704).")
- p.add_argument("--width", type=int, default=1280, help="Calibration video width (Wan2.2 TI2V-5B native: 1280).")
- p.add_argument(
- "--num-frames",
- type=int,
- default=49,
- help="Frames per calibration sample. 49 matches the typical short benchmark; "
- "use 17 to reduce memory pressure during calibration.",
- )
- p.add_argument("--guidance-scale", type=float, default=5.0)
- p.add_argument(
- "--calib-steps",
- type=int,
- default=10,
- help="Denoising steps per calibration prompt (10 is enough for amax statistics).",
- )
- p.add_argument("--calib-size", type=int, default=8, help="How many prompts to use for calibration.")
- p.add_argument("--seed", type=int, default=42)
- p.add_argument(
- "--prompt",
- action="append",
- default=[],
- help="Custom calibration prompt. Repeat to provide multiple.",
- )
- p.add_argument(
- "--quantize-mha",
- action="store_true",
- help="Enable FP8 attention K/V/softmax quantizers. Off by default — Wan2.2's long attention "
- "sequences amplified FP8 drift in the online ablation (see #2920).",
- )
- p.add_argument(
- "--weight-block-size",
- type=str,
- default=None,
- help="Per-block weight quantization as 'M,N' (e.g. '128,128'). Default per-tensor. "
- "Note: vllm-omni's ModelOpt adapter may not yet dispatch block-wise scales — check #2924 "
- "for the HV-1.5 investigation status before relying on this.",
- )
- p.add_argument("--overwrite", action="store_true", help="Replace an existing output directory.")
- return p
-
-
-def _parse_block_size(spec: str | None) -> list[int] | None:
- if spec is None:
- return None
- parts = [int(x) for x in spec.split(",") if x.strip()]
- if len(parts) != 2:
- raise SystemExit(f"--weight-block-size must be 'M,N' (2 ints), got {spec!r}")
- return parts
-
-
-def _require_modelopt() -> Any:
- try:
- import modelopt.torch.quantization as mtq
- except ModuleNotFoundError as exc:
- raise SystemExit(
- "NVIDIA ModelOpt is not installed. Install with:\n"
- " pip install 'nvidia-modelopt[all]'\n"
- f"Original error: {exc}"
- ) from exc
- return mtq
-
-
-def _ensure_paths(args: argparse.Namespace) -> tuple[str, Path]:
- model_path = args.model
- output_dir = Path(args.output).expanduser().resolve()
- if output_dir.exists():
- if not args.overwrite:
- raise SystemExit(f"Output directory already exists: {output_dir}\nPass --overwrite to replace it.")
- shutil.rmtree(output_dir)
- return model_path, output_dir
-
-
-def _select_dtype(name: str) -> torch.dtype:
- return {"bfloat16": torch.bfloat16, "float16": torch.float16}[name]
-
-
-def _build_prompts(args: argparse.Namespace) -> list[str]:
- prompts = args.prompt or DEFAULT_PROMPTS
- if args.calib_size <= 0:
- raise SystemExit("--calib-size must be positive.")
- if len(prompts) < args.calib_size:
- repeats = (args.calib_size + len(prompts) - 1) // len(prompts)
- prompts = (prompts * repeats)[: args.calib_size]
- return prompts[: args.calib_size]
-
-
-# Layers to KEEP at full precision. Wan2.2's module naming:
-# - condition_embedder: time_embedder, time_proj, text_embedder, image_embedder (I2V)
-# - patch_embedding: Conv3dLayer (already not Linear, belt-and-suspenders skip)
-# - scale_shift_table: nn.Parameter modulation (not Linear, but pattern guard)
-# - norm_out: AdaLayerNorm final
-# - proj_out: final nn.Linear
-# - timestep_proj_prepare / output_scale_shift_prepare: SP helpers
-def _filter_func_wan22(name: str) -> bool:
- pattern = re.compile(
- r"(proj_out.*|"
- r".*(condition_embedder|patch_embedding|"
- r"norm_out|scale_shift_table|"
- r"timestep_proj_prepare|output_scale_shift_prepare).*)"
- )
- return pattern.match(name) is not None
-
-
-def _mha_filter_func(name: str) -> bool:
- pattern = re.compile(
- r".*(q_bmm_quantizer|k_bmm_quantizer|v_bmm_quantizer|softmax_quantizer|bmm2_output_quantizer).*"
- )
- return pattern.match(name) is not None
-
-
-def _disable_known_problematic_quantizers(mtq: Any, backbone: torch.nn.Module, *, quantize_mha: bool) -> None:
- if not hasattr(mtq, "disable_quantizer"):
- return
- mtq.disable_quantizer(backbone, _filter_func_wan22)
- if not quantize_mha:
- mtq.disable_quantizer(backbone, _mha_filter_func)
-
-
-def _load_pipeline(model_path: str, dtype: torch.dtype) -> DiffusionPipeline:
- pipe = DiffusionPipeline.from_pretrained(model_path, torch_dtype=dtype)
- if hasattr(pipe, "set_progress_bar_config"):
- pipe.set_progress_bar_config(disable=True)
- pipe.to("cuda")
- return pipe
-
-
-def _build_forward_loop(pipe: DiffusionPipeline, args: argparse.Namespace, prompts: list[str]):
- generator = torch.Generator(device="cuda")
-
- # Try setting guidance on the pipeline's guider if present (newer diffusers APIs).
- guider = getattr(pipe, "guider", None)
- if guider is not None and hasattr(guider, "guidance_scale"):
- try:
- guider.guidance_scale = args.guidance_scale
- except Exception:
- pass
-
- base_kwargs = dict(
- height=args.height,
- width=args.width,
- num_frames=args.num_frames,
- num_inference_steps=args.calib_steps,
- output_type="latent",
- )
-
- def forward_loop(*_unused_args, **_unused_kwargs) -> None:
- with torch.inference_mode():
- for idx, prompt in enumerate(prompts):
- generator.manual_seed(args.seed + idx)
- # Try with guidance_scale first; fall back without on TypeError
- # for pipelines that take CFG via guider config only.
- try:
- pipe(prompt=prompt, generator=generator, guidance_scale=args.guidance_scale, **base_kwargs)
- except TypeError as exc:
- if "guidance_scale" not in str(exc):
- raise
- pipe(prompt=prompt, generator=generator, **base_kwargs)
-
- return forward_loop
-
-
-def _summarize_export(output_dir: Path) -> None:
- cfg_path = output_dir / "transformer" / "config.json"
- if not cfg_path.exists():
- print(f"[warn] {cfg_path} missing.", file=sys.stderr)
- return
- with cfg_path.open(encoding="utf-8") as f:
- cfg = json.load(f)
- qc = cfg.get("quantization_config")
- if not isinstance(qc, dict):
- print("[warn] No quantization_config in transformer/config.json.", file=sys.stderr)
- return
- print("Export summary:")
- print(f" quant_method: {qc.get('quant_method')}")
- print(f" quant_algo: {qc.get('quant_algo')}")
- producer = qc.get("producer")
- if isinstance(producer, dict):
- print(f" producer: {producer.get('name')} {producer.get('version')}")
- print(f" config path: {cfg_path}")
-
-
-def _force_export_quantized_weights(backbone: torch.nn.Module, dtype: torch.dtype) -> int:
- """Convert in-memory weights of quantized modules to actual FP8 storage.
-
- `export_hf_checkpoint` skips this step for unknown model types (Wan2.2 isn't
- in ModelOpt's recognized-model registry), so we must call the per-weight
- export helper ourselves. Same workaround as the HunyuanVideo-1.5 / HunyuanImage-3
- calibration helpers.
- """
- from modelopt.torch.export.quant_utils import (
- QUANTIZATION_NONE,
- get_quantization_format,
- quantizer_attr_names,
- weight_attr_names,
- )
- from modelopt.torch.export.unified_export_hf import _export_quantized_weight
-
- exported = 0
- for name, module in backbone.named_modules():
- try:
- quantization_format = get_quantization_format(module)
- except Exception as exc:
- print(f"[warn] Could not inspect quantization format for {name}: {exc}", file=sys.stderr)
- continue
- if quantization_format == QUANTIZATION_NONE:
- continue
- for weight_name in weight_attr_names(module):
- quantizer_attrs = quantizer_attr_names(weight_name)
- weight_quantizer = getattr(module, quantizer_attrs.weight_quantizer, None)
- if weight_quantizer is None or not getattr(weight_quantizer, "is_enabled", False):
- continue
- _export_quantized_weight(module, dtype, weight_name)
- exported += 1
- return exported
-
-
-def _wan22_quant_config_block(weight_block_size: list[int] | None = None) -> dict:
- """Mirror ModelOpt FP8 metadata expected by vllm-omni's adapter (#2913)."""
- weights_cfg: dict = {"dynamic": False, "num_bits": 8, "type": "float"}
- if weight_block_size is not None:
- weights_cfg["strategy"] = "block"
- weights_cfg["block_structure"] = f"{weight_block_size[0]}x{weight_block_size[1]}"
- return {
- "config_groups": {
- "group_0": {
- "input_activations": {"dynamic": False, "num_bits": 8, "type": "float"},
- "weights": weights_cfg,
- "targets": ["Linear"],
- }
- },
- "ignore": [
- "condition_embedder*",
- "norm_out*",
- "output_scale_shift_prepare*",
- "patch_embedding*",
- "proj_out*",
- "scale_shift_table*",
- "timestep_proj_prepare*",
- ],
- "producer": {"name": "modelopt"},
- "quant_algo": "FP8",
- "quant_method": "modelopt",
- }
-
-
-def _patch_quant_config(output_dir: Path, weight_block_size: list[int] | None = None) -> None:
- """Inject quant_algo: FP8 + config_groups into each transformer's config.json
- so vllm-omni's adapter (#2913) recognises the checkpoint as ModelOpt FP8.
-
- Patches both ``transformer`` and ``transformer_2`` (A14B) when present."""
- for sub in ("transformer", "transformer_2"):
- cfg_path = output_dir / sub / "config.json"
- if not cfg_path.exists():
- continue
- with cfg_path.open(encoding="utf-8") as f:
- cfg = json.load(f)
-
- new_qc = _wan22_quant_config_block(weight_block_size=weight_block_size)
- existing = cfg.get("quantization_config")
- if isinstance(existing, dict):
- producer = existing.get("producer")
- if isinstance(producer, dict):
- new_qc["producer"] = producer
-
- cfg["quantization_config"] = new_qc
- with cfg_path.open("w", encoding="utf-8") as f:
- json.dump(cfg, f, indent=2)
-
-
-def _list_transformers(pipe: DiffusionPipeline) -> list[tuple[str, torch.nn.Module]]:
- """Return [(attr_name, module)] for every DiT on the pipeline.
-
- Wan2.2 A14B has both ``transformer`` (low-noise) and ``transformer_2``
- (high-noise). TI2V-5B has only ``transformer``.
- """
- out = []
- for attr in ("transformer", "transformer_2"):
- mod = getattr(pipe, attr, None)
- if mod is not None:
- out.append((attr, mod))
- return out
-
-
-def _save_pipeline_with_fp8_transformer(
- pipe: DiffusionPipeline,
- model_path: str,
- output_dir: Path,
- max_shard_size: str = "5GB",
-) -> None:
- """Copy source dir verbatim minus transformer dirs, then save each quantized DiT.
-
- Handles A14B's dual transformer (``transformer`` + ``transformer_2``) as well
- as the single-DiT TI2V-5B.
- """
- from modelopt.torch.export.diffusers_utils import hide_quantizers_from_state_dict
-
- src = Path(model_path)
- if not src.exists():
- from huggingface_hub import snapshot_download
-
- src = Path(snapshot_download(model_path))
-
- if output_dir.exists():
- shutil.rmtree(output_dir)
- shutil.copytree(src, output_dir, ignore=shutil.ignore_patterns("transformer", "transformer_2"))
-
- for attr, backbone in _list_transformers(pipe):
- transformer_out = output_dir / attr
- # Pass the nn.Module (transformer), not the Pipeline wrapper.
- with hide_quantizers_from_state_dict(backbone):
- backbone.save_pretrained(
- str(transformer_out),
- safe_serialization=True,
- max_shard_size=max_shard_size,
- )
-
-
-def main() -> None:
- args = _build_parser().parse_args()
- if not torch.cuda.is_available():
- raise SystemExit("CUDA is required for ModelOpt FP8 quantization.")
-
- mtq = _require_modelopt()
- model_path, output_dir = _ensure_paths(args)
- dtype = _select_dtype(args.dtype)
- prompts = _build_prompts(args)
- weight_block_size = _parse_block_size(args.weight_block_size)
-
- print("Quantization plan:")
- print(f" input: {args.model}")
- print(f" output: {output_dir}")
- print(f" dtype: {dtype}")
- print(f" height/width: {args.height}x{args.width}")
- print(f" num_frames: {args.num_frames}")
- print(f" calib_size: {len(prompts)}")
- print(f" calib_steps: {args.calib_steps}")
- print(f" quantize_mha: {args.quantize_mha}")
- print(
- f" weight strategy: {'block-wise ' + str(weight_block_size) if weight_block_size else 'per-tensor (default)'}"
- )
-
- pipe = _load_pipeline(model_path, dtype)
- transformers = _list_transformers(pipe)
- if not transformers:
- raise SystemExit("Pipeline has no transformer or transformer_2 attribute.")
- print(f" found {len(transformers)} transformer(s): {', '.join(a for a, _ in transformers)}")
-
- quant_config = copy.deepcopy(mtq.FP8_DEFAULT_CFG)
- if weight_block_size is not None:
- quant_config["quant_cfg"]["*weight_quantizer"] = {
- "num_bits": (4, 3),
- "block_sizes": {-1: weight_block_size[1], -2: weight_block_size[0]},
- }
- print(
- f" -> overriding weight quantizer with block_sizes={weight_block_size} "
- f"({weight_block_size[0]}x{weight_block_size[1]} tiles)"
- )
-
- forward_loop = _build_forward_loop(pipe, args, prompts)
-
- # Quantize each DiT. The forward_loop runs the full pipeline, so quantizers
- # on every attached transformer (A14B: transformer + transformer_2) get
- # calibrated during the same passes.
- for attr, backbone in transformers:
- print(f"\nQuantizing {attr}...")
- quantized = mtq.quantize(backbone, quant_config, forward_loop)
- if quantized is not None:
- setattr(pipe, attr, quantized)
- _disable_known_problematic_quantizers(mtq, getattr(pipe, attr), quantize_mha=args.quantize_mha)
-
- # Export weights to FP8 only AFTER all transformers are calibrated. Exporting
- # inside the loop would convert transformer's weights to real FP8, and the
- # next transformer's forward_loop (full pipeline) would then fake-quantize an
- # already-FP8 weight -> "fake_e4m3fy not implemented for Float8_e4m3fn".
- print("\nForcing FP8 weight serialization (Wan2.2 isn't in ModelOpt's recognized-model registry,")
- print("so we have to call the per-weight export helper ourselves)...")
- total_exported = 0
- for attr, backbone in _list_transformers(pipe):
- exported = _force_export_quantized_weights(backbone, dtype)
- print(f" -> {exported} weights converted to FP8 in {attr}")
- total_exported += exported
-
- if total_exported == 0:
- raise SystemExit(
- "No quantized weights were exported. Calibration may have skipped every layer "
- "(check the disable_quantizer regex) or `mtq.quantize` did not actually wrap any "
- "weight quantizers."
- )
-
- print("\nSaving pipeline with FP8 transformer(s)...")
- _save_pipeline_with_fp8_transformer(pipe, model_path, output_dir)
- _patch_quant_config(output_dir, weight_block_size=weight_block_size)
- print(f"Saved to: {output_dir}")
- _summarize_export(output_dir)
-
- print("\nNext: validate the checkpoint with vllm-omni:")
- print(
- " python examples/offline_inference/text_to_video/text_to_video.py \\\n"
- f" --model {output_dir} \\\n"
- " --quantization fp8 \\\n"
- " --prompt 'A dog running across a field of golden wheat.' \\\n"
- f" --height {args.height} --width {args.width} --num-frames {args.num_frames} \\\n"
- " --num-inference-steps 30 --guidance-scale 5.0 --seed 42 \\\n"
- " --output outputs/wan22_modelopt_fp8.mp4"
- )
- print(
- "\n (--quantization fp8 is auto-upgraded to ModelOpt FP8 at runtime because the "
- "checkpoint's config.json has modelopt metadata.)"
- )
-
-
-if __name__ == "__main__":
- main()
diff --git a/recipes/README.md b/recipes/README.md
index a6015d9b0..334f44c30 100644
--- a/recipes/README.md
+++ b/recipes/README.md
@@ -19,6 +19,11 @@ add files for this repo. Use one Markdown file per model family by default.
> Individual pages resolve at `https://recipes.vllm.ai//`,
> e.g. [`bosonai/higgs-audio-v3-tts-4b`](https://recipes.vllm.ai/bosonai/higgs-audio-v3-tts-4b).
+The `Recipe` column in the Supported Models documentation links to the
+published recipe when available and otherwise to the corresponding recipe in
+this directory. The recipe files remain the source of truth for commands and
+validation details.
+
Example layout:
```text
diff --git a/tests/docs/test_generate_examples.py b/tests/docs/test_generate_examples.py
index 57d7d4832..34d7045f9 100644
--- a/tests/docs/test_generate_examples.py
+++ b/tests/docs/test_generate_examples.py
@@ -3,8 +3,10 @@
import importlib.util
from pathlib import Path
+from types import SimpleNamespace
import pytest
+import yaml
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -43,3 +45,113 @@ def test_model_display_names_require_string_mapping(tmp_path, monkeypatch):
with pytest.raises(ValueError, match="MODEL_DISPLAY_NAMES_FILE"):
generate_examples.load_model_display_names()
+
+
+def test_quantization_examples_keep_source_urls_under_feature_navigation(tmp_path, monkeypatch):
+ nav_file = tmp_path / "docs" / ".nav.yml"
+ nav_file.parent.mkdir(parents=True)
+ nav_file.write_text(
+ yaml.safe_dump(
+ {
+ "nav": [
+ {
+ "User Guide": [
+ {
+ "Examples": [
+ "examples/README.md",
+ {"Offline Inference": [{"Old": "user_guide/examples/offline_inference/old.md"}]},
+ ]
+ },
+ {"Features": [{"Quantization": [{"Overview": "user_guide/quantization/overview.md"}]}]},
+ ]
+ }
+ ]
+ },
+ sort_keys=False,
+ ),
+ encoding="utf-8",
+ )
+
+ monkeypatch.setattr(generate_examples, "ROOT_DIR", tmp_path)
+ monkeypatch.setattr(generate_examples, "NAV_FILE", nav_file)
+ monkeypatch.setattr(generate_examples, "EXAMPLE_DOC_DIR", tmp_path / "docs/user_guide/examples")
+
+ quantization_example = SimpleNamespace(
+ category="quantization",
+ path=tmp_path / "examples/quantization/check_modelopt_fp8_export.py",
+ nav_title="Check Modelopt FP8 Export",
+ )
+ offline_example = SimpleNamespace(
+ category="offline_inference",
+ path=tmp_path / "examples/offline_inference/demo.py",
+ nav_title="Demo",
+ )
+
+ generate_examples.update_nav_file([quantization_example, offline_example])
+ generate_examples.update_nav_file([quantization_example, offline_example])
+
+ nav_data = yaml.safe_load(nav_file.read_text(encoding="utf-8"))
+ user_guide = nav_data["nav"][0]["User Guide"]
+ examples = next(item["Examples"] for item in user_guide if "Examples" in item)
+ features = next(item["Features"] for item in user_guide if "Features" in item)
+ quantization = next(item["Quantization"] for item in features if "Quantization" in item)
+
+ assert not any(isinstance(item, dict) and "Quantization" in item for item in examples)
+ generated_examples = [
+ item[generate_examples.GENERATED_QUANTIZATION_EXAMPLES_TITLE]
+ for item in quantization
+ if generate_examples.GENERATED_QUANTIZATION_EXAMPLES_TITLE in item
+ ]
+ assert len(generated_examples) == 1
+ assert generated_examples[0] == [
+ {"Check Modelopt FP8 Export": "user_guide/examples/quantization/check_modelopt_fp8_export.md"}
+ ]
+
+ # A later build without quantization examples must not leave a stale
+ # generated group in the hand-authored Quantization section.
+ generate_examples.update_nav_file([offline_example])
+ nav_data = yaml.safe_load(nav_file.read_text(encoding="utf-8"))
+ user_guide = nav_data["nav"][0]["User Guide"]
+ features = next(item["Features"] for item in user_guide if "Features" in item)
+ quantization = next(item["Quantization"] for item in features if "Quantization" in item)
+ assert not any(
+ isinstance(item, dict) and generate_examples.GENERATED_QUANTIZATION_EXAMPLES_TITLE in item
+ for item in quantization
+ )
+
+
+def test_quantization_examples_are_not_added_to_examples_without_feature_target(tmp_path, monkeypatch, caplog):
+ nav_file = tmp_path / "docs" / ".nav.yml"
+ nav_file.parent.mkdir(parents=True)
+ nav_file.write_text(
+ yaml.safe_dump(
+ {
+ "nav": [
+ {
+ "User Guide": [
+ {"Examples": ["examples/README.md"]},
+ ]
+ }
+ ]
+ },
+ sort_keys=False,
+ ),
+ encoding="utf-8",
+ )
+
+ monkeypatch.setattr(generate_examples, "ROOT_DIR", tmp_path)
+ monkeypatch.setattr(generate_examples, "NAV_FILE", nav_file)
+ monkeypatch.setattr(generate_examples, "EXAMPLE_DOC_DIR", tmp_path / "docs/user_guide/examples")
+
+ quantization_example = SimpleNamespace(
+ category="quantization",
+ path=tmp_path / "examples/quantization/check_modelopt_fp8_export.py",
+ nav_title="Check Modelopt FP8 Export",
+ )
+
+ generate_examples.update_nav_file([quantization_example])
+
+ nav_data = yaml.safe_load(nav_file.read_text(encoding="utf-8"))
+ examples = nav_data["nav"][0]["User Guide"][0]["Examples"]
+ assert not any(isinstance(item, dict) and "Quantization" in item for item in examples)
+ assert "omitting generated quantization examples from navigation" in caplog.text