diff --git a/docs/user_guide/quantization/mxfp8.md b/docs/user_guide/quantization/mxfp8.md new file mode 100644 index 000000000..418aad369 --- /dev/null +++ b/docs/user_guide/quantization/mxfp8.md @@ -0,0 +1,187 @@ +# W8A8 MXFP8 Quantization + +## Overview + +W8A8 MXFP8 (Microscaling FP8) quantizes both weights and activations to FP8 +using the OCP MX format: groups of 32 K-dimension elements share a single +`float8_e8m0fnu` exponent scale. This gives better accuracy than channel-wise +FP8 while keeping the same 8-bit weight footprint. + +This method supports two modes: + +| Mode | Description | +|------|-------------| +| **Online** | BF16 weights are quantized to MXFP8 at load time — no pre-processing needed | +| **Offline** | msModelSlim-exported MXFP8 weights converted to diffusers format via `merge_mxfp8_checkpoint.py` — weights and scales are loaded directly from the preprocessed checkpoint | + +## Hardware Support + +| Device | Support | +|--------|---------| +| NVIDIA Blackwell GPU (SM 100+) | ⭕ | +| NVIDIA Ada/Hopper GPU (SM 89+) | ⭕ | +| NVIDIA Ampere GPU (SM 80+) | ⭕ | +| AMD ROCm | ⭕ | +| Intel XPU | ⭕ | +| Ascend NPU (Atlas 950 A5) | ✅ | + +Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this +guide. + +## Model Type Support + +### Diffusion Model (Wan2.2) + +| Model | Mode | Notes | +|-------|------|-------| +| Wan2.2-T2V-A14B | Online + Offline | MoE cascade; quantizes two transformers (`transformer` + `transformer_2`) | +| Wan2.2-I2V-A14B | Online + Offline | MoE cascade; quantizes two transformers (`transformer` + `transformer_2`) | +| Wan2.2-TI2V-5B | Online + Offline | Single transformer | + +### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS) + +| Model | Status | Notes | +|-------|--------|-------| +| Qwen3-Omni | Not validated | — | +| Qwen3-TTS | Not validated | — | + +### Multi-Stage Diffusion Model (BAGEL, GLM-Image) + +| Model | Status | Notes | +|-------|--------|-------| +| BAGEL | Not validated | — | +| GLM-Image | Not validated | — | + +## Configuration + +### Online Mode + +Online mode requires no pre-processing. vLLM-Omni quantizes BF16 weights to +MXFP8 at load time. + +Python API: + +```python +from vllm_omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +omni = Omni(model="", quantization="mxfp8") + +outputs = omni.generate( + "A cat sitting on a windowsill", + OmniDiffusionSamplingParams(num_inference_steps=50), +) +``` + +CLI: + +```bash +python text_to_video.py --model --quantization mxfp8 + +# Online serving +vllm serve --omni --quantization mxfp8 +``` + +### Offline Mode + +Offline mode loads a pre-quantized checkpoint from msModelSlim. A preprocessing +step converts the raw quantized output to the diffusers format expected by +vLLM-Omni and injects the quantization config into `transformer/config.json` so +that vLLM-Omni auto-detects the offline path without a `--quantization` flag. + +#### Step 1 — Quantize with msModelSlim + +```bash +msmodelslim quant \ + --model_path /path/to/Wan2.2-TI2V-5B-Diffusers \ + --save_path /path/to/wan2_2_ti2v_quantized_raw \ + --device npu \ + --model_type Wan2_2 \ + --config_path /path/to/wan2_2_w8a8f8_mxfp.yaml \ + --trust_remote_code True +``` + +After this step, `--save_path` contains the raw quantized safetensors files and +a metadata JSON (`quant_model_description*.json`). + +For cascade MoE models (T2V-A14B, I2V-A14B), msModelSlim outputs two +subdirectories: `high_noise_model/` and `low_noise_model/`. + +#### Step 2 — Preprocess with merge_mxfp8_checkpoint.py + +The script (`vllm_omni/quantization/tools/merge_mxfp8_checkpoint.py`): + +1. Copies the original diffusers model to `--output-path` (VAE, text encoder, + scheduler, etc. are preserved). +2. Remaps tensor names from msModelSlim convention to diffusers convention. +3. Saves the converted weights as `diffusion_pytorch_model.safetensors`. +4. Copies the original `transformer/config.json` and injects + `quantization_config` so that vLLM-Omni auto-detects offline MXFP8. + +For cascade MoE models, steps 2–4 run separately for `high_noise_model/` → +`transformer/` and `low_noise_model/` → `transformer_2/`. + +```bash +python vllm_omni/quantization/tools/merge_mxfp8_checkpoint.py \ + --model-type Wan2.2-TI2V-5B \ + --original-model /path/to/Wan2.2-TI2V-5B-Diffusers \ + --quant-path /path/to/wan2_2_ti2v_quantized_raw \ + --output-path /path/to/Wan2.2-TI2V-5B-MXFP8 +``` + +| Argument | Description | +|----------|-------------| +| `--model-type` | Model variant: `Wan2.2-T2V-A14B`, `Wan2.2-I2V-A14B`, or `Wan2.2-TI2V-5B` | +| `--original-model` | Root directory of the original BF16 diffusers model | +| `--quant-path` | Root directory of the msModelSlim quantized output | +| `--output-path` | Output directory for the merged model (created by the script) | + +The script outputs a complete diffusers model directory at `--output-path`, +with each transformer subfolder containing: + +- `diffusion_pytorch_model.safetensors` — converted FP8 weights +- `config.json` — original transformer config with `quantization_config` injected +- `quant_model_description.json` — renamed quantization metadata (reference only) + +#### Step 3 — Serve + +```bash +python text_to_video.py --model /path/to/Wan2.2-TI2V-5B-MXFP8 + +# Online serving +vllm serve /path/to/Wan2.2-TI2V-5B-MXFP8 --omni +``` + +Python API: + +```python +omni = Omni(model="/path/to/Wan2.2-TI2V-5B-MXFP8") +``` + +!!! note + No `--quantization` flag is needed for offline mode. The preprocessing + script injects `quantization_config` into each `transformer/config.json`, + which vLLM-Omni reads automatically to activate the offline MXFP8 method. + +## Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `method` | str | — | Must be `"mxfp8"` | +| `is_checkpoint_mxfp8_serialized` | bool | `False` | `True` for offline pre-quantized checkpoints; auto-set from `config.json` when using the preprocessing script | +| `ignored_layers` | list[str] | `[]` | Layer name substrings to keep in BF16 (e.g. `"to_out"` matches `blocks.0.attn1.to_out.0`) | + +## Validation and Notes + +1. Online mode quantizes BF16 weights at load time using + `npu_dynamic_mx_quant`. This adds a one-time overhead on the first load + but requires no checkpoint preparation. +2. Offline mode loads FP8 weights directly from the checkpoint. Scales are + stored as `uint8` bytes in safetensors (same bit layout as + `float8_e8m0fnu`) and are reinterpreted at load time without a dtype + conversion. +3. If the offline checkpoint was produced with the old `merge_mxfp8_checkpoint.py` + interface (arguments `--quant-dir`, `--orig-dir`, `--meta-json`, + `--output-dir`), regenerate it with the current script. The old script + wrote a separate `quantization_config.json` that is not read by vLLM-Omni; + the current script injects the config directly into `transformer/config.json`. diff --git a/docs/user_guide/quantization/overview.md b/docs/user_guide/quantization/overview.md index d4e3ce377..c4605c8c2 100644 --- a/docs/user_guide/quantization/overview.md +++ b/docs/user_guide/quantization/overview.md @@ -9,20 +9,20 @@ type has a different quantization scope. | Mode | Guide | Description | Methods | |------|-------|-------------|---------| -| Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8 | +| Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8, MXFP8 W8A8 | | Runtime attention quantization | [Quantized KV Cache](quantized_kvcache.md) | vLLM-Omni dynamically quantizes eligible diffusion Flash Attention tensors during inference. | FP8 FA | -| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, GGUF, AutoRound, msModelSlim, serialized Int8 | +| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, GGUF, AutoRound, msModelSlim, serialized Int8, offline MXFP8 | ## Hardware Support -| Device | FP8 W8A8 | Int8 W8A8 | ModelOpt | GGUF | AutoRound | msModelSlim | -|--------|----------|-----------|----------|------|-----------|-------------| -| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | -| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | -| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ⭕ | ✅ | ✅ | ❌ | -| AMD ROCm | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ❌ | -| Intel XPU | ⭕ | ⭕ | ⭕ | ⭕ | ✅ | ❌ | -| Ascend NPU | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | +| Device | FP8 W8A8 | Int8 W8A8 | ModelOpt | MXFP8 W8A8 | GGUF | AutoRound | msModelSlim | +|--------|----------|-----------|----------|------------|------|-----------|-------------| +| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ⭕ | ✅ | ✅ | ❌ | +| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ⭕ | ✅ | ✅ | ❌ | +| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ⭕ | ⭕ | ✅ | ✅ | ❌ | +| AMD ROCm | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ❌ | +| Intel XPU | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ✅ | ❌ | +| Ascend NPU | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this guide. FP8 on Ampere may use a weight-only path where available. @@ -41,6 +41,7 @@ otherwise. | FP8 W8A8 | [FP8](fp8.md) | Online W8A8 or checkpoint FP8 | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image family and other DiT models | | Int8 W8A8 | [Int8](int8.md) | Online or serialized W8A8 | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image and Z-Image | | ModelOpt | [ModelOpt](modelopt.md) | Pre-quantized FP8 checkpoints | Qwen-Image, Z-Image, FLUX.2, HunyuanImage-3.0 | Validated for ModelOpt FP8 diffusion checkpoints | +| MXFP8 W8A8 | [MXFP8](mxfp8.md) | Online W8A8 or offline pre-quantized | Wan2.2-T2V-A14B, I2V-A14B, TI2V-5B | Ascend NPU only; validated for Wan2.2 | | GGUF | [GGUF](gguf.md) | Pre-quantized transformer weights | Qwen-Image | Validated where a model-specific GGUF adapter exists | | AutoRound | [AutoRound](autoround.md) | Pre-quantized W4A16 checkpoints | FLUX.1-dev; Qwen-Image/Wan2.2 not validated | Checkpoint-driven | | msModelSlim | [msModelSlim](msmodelslim.md) | Pre-quantized Ascend checkpoints | Wan2.2 recipe; HunyuanImage-3.0 inference target | Ascend/NPU path | @@ -56,6 +57,7 @@ in BF16 unless the model guide explicitly adds support. |--------|-------|-------|----------------|--------| | ModelOpt | [ModelOpt](modelopt.md) | Thinker or language-model checkpoint config | Qwen3-Omni thinker | ModelOpt checkpoint path | | Int8 | [Int8](int8.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated | +| MXFP8 | [MXFP8](mxfp8.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated | | GGUF | [GGUF](gguf.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated | | AutoRound | [AutoRound](autoround.md) | Thinker or language-model checkpoint config | Qwen2.5-Omni, Qwen3-Omni | Supported through AutoRound checkpoints | | msModelSlim | [msModelSlim](msmodelslim.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated | @@ -70,6 +72,7 @@ attached to the intended stage rather than applied globally. | FP8 | [FP8](fp8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Requires model-specific validation | | Int8 | [Int8](int8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Requires model-specific validation | | ModelOpt | [ModelOpt](modelopt.md) | Checkpoint-defined diffusion stage | BAGEL, GLM-Image | Requires model-specific validation | +| MXFP8 | [MXFP8](mxfp8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Not validated | | GGUF | [GGUF](gguf.md) | Stage-specific transformer weights | BAGEL, GLM-Image | No validated adapter listed | | AutoRound | [AutoRound](autoround.md) | Checkpoint-defined stage | BAGEL, GLM-Image | No validated checkpoint listed | | msModelSlim | [msModelSlim](msmodelslim.md) | Ascend-generated stage weights | GLM-Image | Requires model-specific adaptation | diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index 386afea31..84fbf2a94 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -218,6 +218,13 @@ def parse_args() -> argparse.Namespace: "Default 1 means pure sharding (no replication). " ), ) + parser.add_argument( + "--quantization", + type=str, + default=None, + choices=["fp8", "mxfp8", "int8", "gguf"], + help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).", + ) parser.add_argument( "--enable-diffusion-pipeline-profiler", action="store_true", @@ -320,7 +327,7 @@ def main(): hsdp_shard_size=args.hsdp_shard_size, hsdp_replicate_size=args.hsdp_replicate_size, ) - omni = Omni( + omni_kwargs = dict( model=args.model, enable_layerwise_offload=args.enable_layerwise_offload, vae_use_slicing=args.vae_use_slicing, @@ -339,6 +346,9 @@ def main(): enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler, profiler_config=args.profiler_config, ) + if args.quantization is not None: + omni_kwargs["quantization"] = args.quantization + omni = Omni(**omni_kwargs) if profiler_enabled: print("[Profiler] Starting profiling...") diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index 1dba97c20..b19f0095e 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -206,8 +206,8 @@ def parse_args() -> argparse.Namespace: "--quantization", type=str, default=None, - choices=["fp8", "gguf"], - help="Quantization method for the transformer (fp8 for online FP8 quantization).", + choices=["fp8", "mxfp8", "int8", "gguf"], + help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).", ) parser.add_argument( "--use-hsdp", diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py index ff21997c9..e3286cf4f 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py @@ -15,6 +15,7 @@ import torch from diffusers.utils.torch_utils import randn_tensor from torch import nn from transformers import AutoTokenizer, UMT5EncoderModel +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig @@ -118,9 +119,12 @@ def load_transformer_config(model_path: str, subfolder: str = "transformer", loc return {} -def create_transformer_from_config(config: dict) -> WanTransformer3DModel: +def create_transformer_from_config( + config: dict, + quant_config: QuantizationConfig | None = None, +) -> WanTransformer3DModel: """Create WanTransformer3DModel from config dict.""" - kwargs = {} + kwargs: dict = {} if "patch_size" in config: kwargs["patch_size"] = tuple(config["patch_size"]) @@ -153,6 +157,58 @@ def create_transformer_from_config(config: dict) -> WanTransformer3DModel: if "pos_embed_seq_len" in config: kwargs["pos_embed_seq_len"] = config["pos_embed_seq_len"] + # Auto-detect quantization from transformer's config.json when not explicitly provided. + # merge_mxfp8_checkpoint.py injects quantization_config into config.json so that + # offline quantized checkpoints are recognized here without a CLI flag. + if "quantization_config" in config: + from vllm_omni.quantization.factory import build_quant_config + + disk_qc = config["quantization_config"] + if isinstance(disk_qc, dict) and "quant_method" in disk_qc: + qc_method = disk_qc["quant_method"] + qc_kwargs = {k: v for k, v in disk_qc.items() if k != "quant_method"} + if quant_config is None: + # No CLI flag: full auto-detection. + quant_config = build_quant_config(qc_method, **qc_kwargs) + logger.info( + "Auto-detected quantization from transformer config.json: method=%s kwargs=%s", + qc_method, + qc_kwargs, + ) + elif quant_config.get_name() != qc_method: + # The caller supplied a quant_config of a different method than + # what the checkpoint was built with. Loading serialized tensors + # (e.g. MXFP8 weight scales) with the wrong linear method would + # produce corrupt output or a shape mismatch crash. Reject early + # so the user gets a clear message instead of a silent failure. + raise ValueError( + f"Checkpoint config.json declares quant_method={qc_method!r} but the " + f"active quantization config is {quant_config.get_name()!r}. " + "Pass a matching --quantization flag or omit it for auto-detection." + ) + elif ( + qc_kwargs.get("is_checkpoint_mxfp8_serialized", False) + and hasattr(quant_config, "is_checkpoint_mxfp8_serialized") + and not quant_config.is_checkpoint_mxfp8_serialized + ): + # Same method: CLI provided online mode but config.json marks this + # as a pre-quantized offline checkpoint. Switch to offline mode so + # users can pass --quantization mxfp8 without knowing the + # online/offline distinction. + quant_config = build_quant_config(qc_method, **qc_kwargs) + logger.info( + "config.json marks checkpoint as serialized; switching from online to offline MXFP8 mode.", + ) + elif isinstance(disk_qc, str) and quant_config is None: + quant_config = build_quant_config(disk_qc) + logger.info( + "Auto-detected quantization from transformer config.json: method=%s", + disk_qc, + ) + + if quant_config is not None: + kwargs["quant_config"] = quant_config + return WanTransformer3DModel(**kwargs) @@ -371,8 +427,9 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe ) def _create_transformer(self, config: dict) -> WanTransformer3DModel: - """Create a transformer from a config dict. Subclasses may override.""" - return create_transformer_from_config(config) + """Create a transformer from a config dict. Respects od_config.quantization_config.""" + quant_config = getattr(self.od_config, "quantization_config", None) + return create_transformer_from_config(config, quant_config=quant_config) @property def guidance_scale(self): diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py index 67fbdd3f0..0651356cc 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py @@ -37,6 +37,7 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import ( resolve_wan_sample_solver, retrieve_latents, ) +from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel from vllm_omni.diffusion.postprocess import interpolate_video_tensor from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin from vllm_omni.diffusion.request import OmniDiffusionRequest @@ -223,10 +224,10 @@ class Wan22I2VPipeline( # Transformers (weights loaded via load_weights) # Load config from model directory or HF Hub to get correct in_channels for I2V models transformer_config = load_transformer_config(model, "transformer", local_files_only) - self.transformer = create_transformer_from_config(transformer_config) + self.transformer = self._create_transformer(transformer_config) if self.has_transformer_2: transformer_2_config = load_transformer_config(model, "transformer_2", local_files_only) - self.transformer_2 = create_transformer_from_config(transformer_2_config) + self.transformer_2 = self._create_transformer(transformer_2_config) else: self.transformer_2 = None @@ -367,6 +368,11 @@ class Wan22I2VPipeline( image_embeds = self.image_encoder(pixel_values, output_hidden_states=True) return image_embeds.hidden_states[-2] + def _create_transformer(self, config: dict) -> WanTransformer3DModel: + """Create a transformer from a config dict. Respects od_config.quantization_config.""" + quant_config = getattr(self.od_config, "quantization_config", None) + return create_transformer_from_config(config, quant_config=quant_config) + def forward( self, req: OmniDiffusionRequest, diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py index eec5c9cbf..360eb83b9 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py @@ -46,6 +46,7 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import ( resolve_wan_sample_solver, retrieve_latents, ) +from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel from vllm_omni.diffusion.postprocess import interpolate_video_tensor from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.inputs.data import OmniTextPrompt @@ -199,7 +200,7 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress # Single transformer (TI2V uses dense 5B model, not MoE) # Load config from model to get correct dimensions transformer_config = load_transformer_config(model, "transformer", local_files_only) - self.transformer = create_transformer_from_config(transformer_config) + self.transformer = self._create_transformer(transformer_config) self._sample_solver = "unipc" self._flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0 @@ -216,6 +217,11 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress self._num_timesteps = None self._current_timestep = None + def _create_transformer(self, config: dict) -> WanTransformer3DModel: + """Create a transformer from a config dict. Respects od_config.quantization_config.""" + quant_config = getattr(self.od_config, "quantization_config", None) + return create_transformer_from_config(config, quant_config=quant_config) + @property def guidance_scale(self): return self._guidance_scale diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py index 9de620400..75bdac27f 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py @@ -21,6 +21,7 @@ from dataclasses import replace import PIL.Image import torch from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig @@ -41,7 +42,10 @@ from vllm_omni.platforms import current_omni_platform logger = init_logger(__name__) -def create_vace_transformer_from_config(config: dict) -> WanVACETransformer3DModel: +def create_vace_transformer_from_config( + config: dict, + quant_config: QuantizationConfig | None = None, +) -> WanVACETransformer3DModel: """Create WanVACETransformer3DModel from config dict.""" kwargs = {} if "patch_size" in config: @@ -78,6 +82,8 @@ def create_vace_transformer_from_config(config: dict) -> WanVACETransformer3DMod kwargs["vace_layers"] = config["vace_layers"] if "vace_in_channels" in config: kwargs["vace_in_channels"] = config["vace_in_channels"] + if quant_config is not None: + kwargs["quant_config"] = quant_config return WanVACETransformer3DModel(**kwargs) @@ -174,8 +180,9 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput): super().__init__(od_config=od_config, prefix=prefix) def _create_transformer(self, config: dict) -> WanVACETransformer3DModel: - """Build VACE transformer directly from config dict.""" - return create_vace_transformer_from_config(config) + """Build VACE transformer. Respects od_config.quantization_config.""" + quant_config = getattr(self.od_config, "quantization_config", None) + return create_vace_transformer_from_config(config, quant_config=quant_config) def diffuse( self, diff --git a/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py b/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py index 2015d3272..b2dcec444 100644 --- a/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py +++ b/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py @@ -19,6 +19,7 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.conv import Conv3dLayer from vllm.model_executor.layers.linear import ColumnParallelLinear, QKVParallelLinear, RowParallelLinear +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata @@ -72,7 +73,16 @@ class DistributedRMSNorm(nn.Module): class ColumnParallelGELU(nn.Module): """Column parallel linear with GELU activation.""" - def __init__(self, dim_in: int, dim_out: int, *, approximate: str = "tanh", bias: bool = True): + def __init__( + self, + dim_in: int, + dim_out: int, + *, + approximate: str = "tanh", + bias: bool = True, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ): super().__init__() self.proj = ColumnParallelLinear( dim_in, @@ -80,6 +90,8 @@ class ColumnParallelGELU(nn.Module): bias=bias, gather_output=False, return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.proj" if prefix else "proj", ) self.approximate = approximate @@ -100,12 +112,21 @@ class WanFeedForward(nn.Module): inner_dim: int, dim_out: int | None = None, bias: bool = True, + quant_config: QuantizationConfig | None = None, + prefix: str = "", ) -> None: super().__init__() dim_out = dim_out or dim # ColumnParallel: scatter to each tp_rank - self.net_0 = ColumnParallelGELU(dim, inner_dim, approximate="tanh", bias=bias) + self.net_0 = ColumnParallelGELU( + dim, + inner_dim, + approximate="tanh", + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.net_0" if prefix else "net_0", + ) # Placeholder for weight loading compatibility self.net_1 = nn.Identity() # RowParallel: gather from each tp_rank @@ -115,6 +136,8 @@ class WanFeedForward(nn.Module): bias=bias, input_is_parallel=True, return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.net_2" if prefix else "net_2", ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -329,6 +352,7 @@ class WanSelfAttention(nn.Module): head_dim: int, eps: float = 1e-5, dropout: float = 0.0, + quant_config: QuantizationConfig | None = None, prefix: str = "", ): super().__init__() @@ -344,6 +368,8 @@ class WanSelfAttention(nn.Module): head_size=head_dim, total_num_heads=num_heads, bias=True, + quant_config=quant_config, + prefix=f"{prefix}.to_qkv" if prefix else "to_qkv", ) self.num_heads = self.to_qkv.num_heads @@ -364,6 +390,8 @@ class WanSelfAttention(nn.Module): bias=True, input_is_parallel=True, return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.to_out" if prefix else "to_out", ) self.dropout = nn.Dropout(dropout) @@ -433,6 +461,7 @@ class WanCrossAttention(nn.Module): eps: float = 1e-5, dropout: float = 0.0, added_kv_proj_dim: int | None = None, + quant_config: QuantizationConfig | None = None, prefix: str = "", ): super().__init__() @@ -450,6 +479,8 @@ class WanCrossAttention(nn.Module): bias=True, gather_output=False, return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.to_q" if prefix else "to_q", ) # Separate K and V projections for cross-attention @@ -459,6 +490,8 @@ class WanCrossAttention(nn.Module): bias=True, gather_output=False, return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.to_k" if prefix else "to_k", ) self.to_v = ColumnParallelLinear( @@ -467,6 +500,8 @@ class WanCrossAttention(nn.Module): bias=True, gather_output=False, return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.to_v" if prefix else "to_v", ) tp_size = get_tensor_model_parallel_world_size() @@ -490,6 +525,8 @@ class WanCrossAttention(nn.Module): bias=True, gather_output=False, return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.add_k_proj" if prefix else "add_k_proj", ) self.add_v_proj = ColumnParallelLinear( added_kv_proj_dim, @@ -497,6 +534,8 @@ class WanCrossAttention(nn.Module): bias=True, gather_output=False, return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.add_v_proj" if prefix else "add_v_proj", ) if get_tensor_model_parallel_world_size() > 1: self.norm_added_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps) @@ -514,6 +553,8 @@ class WanCrossAttention(nn.Module): bias=True, input_is_parallel=True, return_bias=False, + quant_config=quant_config, + prefix=f"{prefix}.to_out" if prefix else "to_out", ) self.dropout = nn.Dropout(dropout) @@ -606,6 +647,7 @@ class WanTransformerBlock(nn.Module): eps: float = 1e-6, added_kv_proj_dim: int | None = None, cross_attn_norm: bool = False, + quant_config: QuantizationConfig | None = None, prefix: str = "", ): super().__init__() @@ -619,7 +661,8 @@ class WanTransformerBlock(nn.Module): num_heads=num_heads, head_dim=head_dim, eps=eps, - prefix=f"{prefix}.attn1", + quant_config=quant_config, + prefix=f"{prefix}.attn1" if prefix else "attn1", ) # 2. Cross-attention @@ -629,12 +672,19 @@ class WanTransformerBlock(nn.Module): head_dim=head_dim, eps=eps, added_kv_proj_dim=added_kv_proj_dim, - prefix=f"{prefix}.attn2", + quant_config=quant_config, + prefix=f"{prefix}.attn2" if prefix else "attn2", ) self.norm2 = LayerNorm(dim, eps, elementwise_affine=True) if cross_attn_norm else nn.Identity() # 3. Feed-forward - self.ffn = WanFeedForward(dim=dim, inner_dim=ffn_dim, dim_out=dim) + self.ffn = WanFeedForward( + dim=dim, + inner_dim=ffn_dim, + dim_out=dim, + quant_config=quant_config, + prefix=f"{prefix}.ffn" if prefix else "ffn", + ) self.norm3 = AdaLayerNorm(dim, elementwise_affine=False, eps=eps) # Scale-shift table for modulation @@ -791,6 +841,7 @@ class WanTransformer3DModel(nn.Module): added_kv_proj_dim: int | None = None, rope_max_seq_len: int = 1024, pos_embed_seq_len: int | None = None, + quant_config: QuantizationConfig | None = None, ): super().__init__() @@ -849,6 +900,7 @@ class WanTransformer3DModel(nn.Module): eps, added_kv_proj_dim, cross_attn_norm, + quant_config=quant_config, prefix=f"blocks.{layer_idx}", ) for layer_idx in range(num_layers) @@ -1015,11 +1067,15 @@ class WanTransformer3DModel(nn.Module): original_name = name lookup_name = name - # Handle QKV fusion + # Handle QKV fusion for weight tensors (separate to_q/k/v → fused to_qkv). + # Pre-fused to_qkv tensors (from offline MXFP8 merged checkpoint) fall + # through to the else branch and are loaded directly. for param_name, weight_name, shard_id in stacked_params_mapping: if weight_name not in original_name: continue lookup_name = original_name.replace(weight_name, param_name) + if lookup_name not in params_dict: + break param = params_dict[lookup_name] weight_loader = param.weight_loader weight_loader(param, loaded_weight, shard_id) diff --git a/vllm_omni/diffusion/models/wan2_2/wan2_2_vace_transformer.py b/vllm_omni/diffusion/models/wan2_2/wan2_2_vace_transformer.py index 995b16c48..3f66a0d1a 100644 --- a/vllm_omni/diffusion/models/wan2_2/wan2_2_vace_transformer.py +++ b/vllm_omni/diffusion/models/wan2_2/wan2_2_vace_transformer.py @@ -10,6 +10,7 @@ from typing import Any import torch import torch.nn as nn import torch.nn.functional as F +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelInput from vllm_omni.diffusion.distributed.sp_sharding import sp_shard @@ -33,6 +34,7 @@ class VaceWanTransformerBlock(WanTransformerBlock): added_kv_proj_dim: int | None = None, cross_attn_norm: bool = False, block_id: int = 0, + quant_config: QuantizationConfig | None = None, prefix: str = "", ): super().__init__( @@ -42,6 +44,7 @@ class VaceWanTransformerBlock(WanTransformerBlock): eps, added_kv_proj_dim, cross_attn_norm, + quant_config=quant_config, prefix=prefix, ) self.proj_in = nn.Linear(dim, dim) if block_id == 0 else None @@ -92,9 +95,10 @@ class WanVACETransformer3DModel(WanTransformer3DModel): *, vace_layers: list[int] | None = None, vace_in_channels: int | None = None, + quant_config: QuantizationConfig | None = None, **kwargs, ): - super().__init__(**kwargs) + super().__init__(quant_config=quant_config, **kwargs) self.vace_blocks = None self.vace_patch_embedding = None @@ -127,6 +131,7 @@ class WanVACETransformer3DModel(WanTransformer3DModel): self.config.added_kv_proj_dim, self.config.cross_attn_norm, block_id=i, + quant_config=quant_config, prefix=f"vace_blocks.{i}", ) for i in range(len(vace_layers)) diff --git a/vllm_omni/quantization/__init__.py b/vllm_omni/quantization/__init__.py index a709dc74d..d8cad3a98 100644 --- a/vllm_omni/quantization/__init__.py +++ b/vllm_omni/quantization/__init__.py @@ -15,8 +15,10 @@ from .component_config import ComponentQuantizationConfig from .factory import SUPPORTED_QUANTIZATION_METHODS, build_quant_config from .inc_config import OmniINCConfig -# DiffusionGGUFConfig is NOT imported here to avoid pulling in -# GGUF -> fused_moe -> pynvml at module load time. +# Heavy configs (GGUF, MXFP8) are NOT imported here to avoid pulling in +# optional dependencies (pynvml, torch_npu) at module load time. +# Import them directly when needed: +# from vllm_omni.quantization.mxfp8_config import DiffusionMXFP8Config __all__ = [ "build_quant_config", diff --git a/vllm_omni/quantization/factory.py b/vllm_omni/quantization/factory.py index de9e6033b..ea7b8a7fe 100644 --- a/vllm_omni/quantization/factory.py +++ b/vllm_omni/quantization/factory.py @@ -41,6 +41,13 @@ def _build_int8(**kw: Any) -> QuantizationConfig: return DiffusionInt8Config(**kw) +def _build_mxfp8(**kw: Any) -> QuantizationConfig: + """Lazy import for W8A8 MXFP8 diffusion config (NPU only).""" + from .mxfp8_config import DiffusionMXFP8Config + + return DiffusionMXFP8Config(**kw) + + def _build_inc(**kw: Any) -> QuantizationConfig: """Lazy import for INC/AutoRound config with checkpoint kwarg normalization.""" from .inc_config import OmniINCConfig @@ -58,6 +65,7 @@ def _build_inc(**kw: Any) -> QuantizationConfig: _OVERRIDES: dict[str, Callable[..., QuantizationConfig]] = { "gguf": _build_gguf, "int8": _build_int8, + "mxfp8": _build_mxfp8, "inc": _build_inc, "auto-round": _build_inc, "auto_round": _build_inc, diff --git a/vllm_omni/quantization/mxfp8_config.py b/vllm_omni/quantization/mxfp8_config.py new file mode 100644 index 000000000..544c6a4ca --- /dev/null +++ b/vllm_omni/quantization/mxfp8_config.py @@ -0,0 +1,452 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""W8A8 MXFP8 (Microscaling FP8) online/offline quantization for diffusion transformers. + +Architecture: + + MXFPLinearMethodBase – platform-agnostic skeleton; defines apply() (reshape only), + _apply_inner() (default single-scale dispatch), and two + abstract ops (_quantize_activation, _quant_matmul). + NPUMxfp8LinearMethod – NPU offline: create_weights for pre-quantized checkpoint, + process_weights normalization, and NPU MXFP8 ops. + NPUMxfp8OnlineLinearMethod – NPU online: _LazyWeightMixin for create_weights, + overrides process_weights to quantize BF16 → FP8. + +Extending to a new platform (e.g. CUDA MXFP8 once the ops are available): + + class CUDAMxfp8LinearMethod(MXFPLinearMethodBase): + def create_weights(self, ...): ... + def process_weights_after_loading(self, layer): ... + def _quantize_activation(self, x): ... # CUDA FP8 quant op + def _quant_matmul(self, ...): ... # CUDA FP8 GEMM + + class CUDAMxfp8OnlineLinearMethod(_LazyWeightMixin, CUDAMxfp8LinearMethod): + def process_weights_after_loading(self, layer): ... # BF16 → FP8 + +Extending to a new MX precision with a different calling convention (e.g. dual-scale FP4): + + class NPUMxfp4DualScaleLinearMethod(MXFPLinearMethodBase): + def _apply_inner(self, layer, x, bias, ori_dtype): ... # override for 3-tuple path + def _quantize_activation(self, x, smooth_scale): ... # returns 3-tuple + def _quant_matmul(self, x_q, l0, l1, layer, ...): ... # dual-level GEMM +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +import torch +from torch.nn import Module +from vllm.logger import init_logger +from vllm.model_executor.layers.linear import ( + LinearBase, + LinearMethodBase, + UnquantizedLinearMethod, +) +from vllm.model_executor.layers.quantization import QuantizationMethods +from vllm.model_executor.layers.quantization.base_config import ( + QuantizationConfig, + QuantizeMethodBase, +) +from vllm.model_executor.layers.quantization.fp8 import CopyNumelCounter, _copy_missing_attrs +from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped +from vllm.model_executor.model_loader.weight_utils import initialize_single_dummy_weight +from vllm.model_executor.parameter import ModelWeightParameter +from vllm.model_executor.utils import replace_parameter + +from vllm_omni.platforms import current_omni_platform + +if TYPE_CHECKING: + from vllm.model_executor.models.utils import WeightsMapper + +logger = init_logger(__name__) + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + + +class DiffusionMXFP8Config(QuantizationConfig): + """W8A8 MXFP8 quantization config for diffusion transformers. + + Supports both online (BF16 checkpoint → quantize at load time) and offline + (pre-quantized MXFP8 checkpoint) modes, mirroring DiffusionInt8Config. + + MX (microscaling) format: groups of 32 K-dimension elements share one + float8_e8m0fnu exponent scale, matching MXFP8 as defined in the OCP MX spec. + """ + + def __init__( + self, + is_checkpoint_mxfp8_serialized: bool = False, + ignored_layers: list[str] | None = None, + ) -> None: + super().__init__() + self.is_checkpoint_mxfp8_serialized = is_checkpoint_mxfp8_serialized + self.ignored_layers = ignored_layers or [] + + @classmethod + def get_name(cls) -> QuantizationMethods: + return "mxfp8" + + @classmethod + def get_supported_act_dtypes(cls) -> list[torch.dtype]: + return [torch.bfloat16, torch.float16] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> list[str]: + return [] + + def apply_vllm_mapper(self, hf_to_vllm_mapper: WeightsMapper) -> None: + if self.ignored_layers: + self.ignored_layers = hf_to_vllm_mapper.apply_list(self.ignored_layers) + + @classmethod + def from_config(cls, config: dict[str, Any]) -> DiffusionMXFP8Config: + is_serialized = cls.get_from_keys_or(config, ["is_checkpoint_mxfp8_serialized"], False) + ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None) + if not ignored_layers: + ignored_layers = cls.get_from_keys_or(config, ["modules_to_not_convert"], None) + return cls( + is_checkpoint_mxfp8_serialized=is_serialized, + ignored_layers=ignored_layers, + ) + + def get_quant_method( + self, + layer: torch.nn.Module, + prefix: str, + ) -> QuantizeMethodBase | None: + if isinstance(layer, LinearBase): + if is_layer_skipped( + prefix=prefix, + ignored_layers=self.ignored_layers, + fused_mapping=self.packed_modules_mapping, + ): + return UnquantizedLinearMethod() + if current_omni_platform.is_npu(): + if self.is_checkpoint_mxfp8_serialized: + return NPUMxfp8LinearMethod(self) + return NPUMxfp8OnlineLinearMethod(self) + # Placeholder for future platforms: add `elif is_cuda(): ...` here. + raise NotImplementedError( + "DiffusionMXFP8Config (W8A8 MXFP8) is currently only supported " + "on NPU (Ascend) platforms. CUDA support is not yet implemented." + ) + return None + + +# --------------------------------------------------------------------------- +# _LazyWeightMixin — shared by all online methods +# --------------------------------------------------------------------------- + + +class _LazyWeightMixin: + """Weight registered on meta device, materialised just-in-time on first load chunk. + + Platform-agnostic; shared by all online MXFP methods. + Imported by mxfp4_config.py to avoid duplication. + """ + + uses_meta_device: bool = True + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + + layer.logical_widths = output_partition_sizes + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.orig_dtype = params_dtype + layer.weight_block_size = None + + def patched_weight_loader(param, loaded_weight, *args, **kwargs): + if not hasattr(layer, "_loaded_numel"): + layer._loaded_numel = 0 + weight = ModelWeightParameter( + data=torch.empty_like(layer.weight, device=layer._load_device), + input_dim=1, + output_dim=0, + weight_loader=patched_weight_loader, + ) + _copy_missing_attrs(layer.weight, weight) + layer.register_parameter("weight", weight) + del layer._load_device + + param = layer.weight + counter = CopyNumelCounter() + with counter: + res = weight_loader(param, loaded_weight, *args, **kwargs) + layer._loaded_numel += counter.copied_numel + + if layer._loaded_numel == layer.weight.numel(): + self.process_weights_after_loading(layer) + layer._already_called_process_weights_after_loading = True + return res + + weight = ModelWeightParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition, + device="meta", + dtype=params_dtype, + ), + input_dim=1, + output_dim=0, + weight_loader=patched_weight_loader, + ) + layer._load_device = torch.get_default_device() + layer.register_parameter("weight", weight) + + +# --------------------------------------------------------------------------- +# Abstract base — platform-agnostic apply skeleton +# --------------------------------------------------------------------------- + + +class MXFPLinearMethodBase(LinearMethodBase, ABC): + """Platform-agnostic MXFP linear method base. + + Defines the apply() skeleton (flatten → _apply_inner → reshape) and three + hooks that subclasses implement: + + _quantize_activation(x) → tuple (arity depends on variant) + _quant_matmul(x_q, x_scale, layer, bias, ori_dtype) → Tensor (single-scale default) + _apply_inner(layer, x, bias, ori_dtype) → Tensor (override for dual-scale) + + Extension guide: + Single-scale (MXFP8, MXFP4): implement _quantize_activation + _quant_matmul only. + Dual-scale (MXFP4 DualScale): override _apply_inner for a different calling convention. + Subclasses must NOT override apply() — reshape logic lives here exclusively. + """ + + @abstractmethod + def _quantize_activation(self, x: torch.Tensor) -> tuple: + """Quantize 2-D activation. Return arity must match what _apply_inner expects.""" + + @abstractmethod + def _quant_matmul( + self, + x_q: torch.Tensor, + x_scale: torch.Tensor, + layer: torch.nn.Module, + bias: torch.Tensor | None, + ori_dtype: torch.dtype, + ) -> torch.Tensor: + """Fused MXFP quantized GEMM. Weight and scale accessed from layer.""" + + def _apply_inner( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None, + ori_dtype: torch.dtype, + ) -> torch.Tensor: + """Default single-scale inner loop: 2-tuple quantize → matmul. + + Override this (not apply()) when a different quantize/matmul convention + is needed, e.g. dual-scale variants that return a 3-tuple from + _quantize_activation and pass extra tensors to _quant_matmul. + """ + x_q, x_scale = self._quantize_activation(x) + return self._quant_matmul(x_q, x_scale, layer, bias, ori_dtype) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + """Shared apply skeleton: reshape → _apply_inner → unreshape. + + Do NOT override this method. Override _apply_inner() instead. + """ + ori_shape = x.shape + ori_dtype = x.dtype + x = x.reshape(-1, ori_shape[-1]) + output = self._apply_inner(layer, x, bias, ori_dtype) + return output.reshape(*ori_shape[:-1], -1) + + +# --------------------------------------------------------------------------- +# NPU MXFP8 offline method (pre-quantized checkpoint) +# --------------------------------------------------------------------------- + + +class NPUMxfp8LinearMethod(MXFPLinearMethodBase): + """NPU W8A8 MXFP8 offline linear method for pre-quantized checkpoints. + + Weight canonical layout after process_weights_after_loading: + weight : (K, N) float8_e4m3fn – pre-transposed for GEMM + weight_scale: (K_groups/2, N, 2) float8_e8m0fnu – reshaped + pre-transposed + + NPUMxfp8OnlineLinearMethod normalises to the same layout so apply() is shared. + """ + + def __init__(self, quant_config: DiffusionMXFP8Config) -> None: + self.quant_config = quant_config + self.out_dtype = torch.get_default_dtype() + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + """Register weight and per-group MX scale for a pre-quantized checkpoint.""" + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + + layer.logical_widths = output_partition_sizes + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.orig_dtype = params_dtype + layer.weight_block_size = None + + layer.register_parameter( + "weight", + ModelWeightParameter( + data=torch.empty(output_size_per_partition, input_size_per_partition, dtype=params_dtype), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ), + ) + + # Scale stored as uint8 in safetensors (float8_e8m0fnu is same bit width). + # Using uint8 avoids a lossy float32 round-trip when loading the checkpoint. + num_groups = (input_size_per_partition + 31) // 32 + layer.register_parameter( + "weight_scale", + ModelWeightParameter( + data=torch.empty(output_size_per_partition, num_groups, dtype=torch.uint8), + input_dim=None, + output_dim=0, + weight_loader=weight_loader, + ), + ) + + def process_weights_after_loading(self, layer: Module) -> None: + """Cast checkpoint weight to FP8 and normalise to canonical GEMM layout.""" + if getattr(layer, "_already_called_process_weights_after_loading", False): + return + + import torch_npu + + # Weight: BF16 → float8_e4m3fn via npu_dtype_cast, then transpose (N,K) → (K,N). + w = layer.weight + if w.dtype != torch_npu.float8_e4m3fn: + w = torch_npu.npu_dtype_cast(w.npu(), torch_npu.float8_e4m3fn) + w = w.transpose(0, 1).contiguous() + + # Scale: checkpoint stores uint8 bytes that ARE float8_e8m0fnu bits. + # Only convert if neither uint8 nor the target NPU dtype already. + # Pad K_groups to even so the (K_groups/2, N, 2) reshape is always valid. + s = layer.weight_scale.data + if s.dtype not in (torch.uint8, torch_npu.float8_e8m0fnu): + s = s.to(torch_npu.float8_e8m0fnu) + N, K_groups = s.shape + if K_groups % 2 == 1: + s = torch.cat([s, torch.zeros(N, 1, dtype=s.dtype, device=s.device)], dim=1) + K_groups += 1 + s = s.reshape(N, K_groups // 2, 2).transpose(0, 1).contiguous() + + replace_parameter(layer, "weight", w) + replace_parameter(layer, "weight_scale", s) + layer._already_called_process_weights_after_loading = True + + # --- NPU MXFP8 ops — shared with online path via inheritance --- + + def _quantize_activation(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + import torch_npu + + return torch_npu.npu_dynamic_mx_quant(x, dst_type=torch_npu.float8_e4m3fn) + + def _quant_matmul( + self, + x_q: torch.Tensor, + x_scale: torch.Tensor, + layer: torch.nn.Module, + bias: torch.Tensor | None, + ori_dtype: torch.dtype, + ) -> torch.Tensor: + import torch_npu + + # NPU npu_quant_matmul requires bias in float32. + if bias is not None and bias.dtype != torch.float32: + bias = bias.to(torch.float32) + return torch_npu.npu_quant_matmul( + x_q, + layer.weight, # (K, N) float8_e4m3fn + layer.weight_scale, # (S/2, N, 2) float8_e8m0fnu + scale_dtype=torch_npu.float8_e8m0fnu, + pertoken_scale=x_scale, + pertoken_scale_dtype=torch_npu.float8_e8m0fnu, + bias=bias, + output_dtype=ori_dtype, + group_sizes=[1, 1, 32], + ) + + +# --------------------------------------------------------------------------- +# NPU MXFP8 online method (BF16 checkpoint → quantize at load time) +# --------------------------------------------------------------------------- + + +class NPUMxfp8OnlineLinearMethod(_LazyWeightMixin, NPUMxfp8LinearMethod): + """NPU W8A8 MXFP8 online linear method. + + MRO: NPUMxfp8OnlineLinearMethod → _LazyWeightMixin → NPUMxfp8LinearMethod + → MXFPLinearMethodBase → LinearMethodBase + + create_weights : _LazyWeightMixin (meta device + patched loader) + process_weights : NPUMxfp8OnlineLinearMethod (BF16 → FP8 + normalize) + apply / ops : NPUMxfp8LinearMethod / MXFPLinearMethodBase (shared) + """ + + def process_weights_after_loading(self, layer: Module) -> None: + if getattr(layer, "_already_called_process_weights_after_loading", False): + return + + import torch_npu + + # Materialise weight if still on meta device (dummy-weight init path). + if layer.weight.device == torch.device("meta"): + weight = ModelWeightParameter( + data=torch.empty_like(layer.weight, device=layer._load_device), + input_dim=1, + output_dim=0, + weight_loader=layer.weight.weight_loader, + ) + _copy_missing_attrs(layer.weight, weight) + layer.register_parameter("weight", weight) + initialize_single_dummy_weight(layer.weight) + + # NPU: quantize BF16/FP16 (N, K) → FP8 (N, K) + MX scale (N, S). + weight_fp8, weight_scale_raw = torch_npu.npu_dynamic_mx_quant(layer.weight, dst_type=torch_npu.float8_e4m3fn) + + # Normalize to canonical layout shared with offline path. + weight_scale = weight_scale_raw.reshape(weight_scale_raw.shape[0], -1, 2).transpose(0, 1).contiguous() + weight_fp8 = weight_fp8.transpose(0, 1).contiguous() + + replace_parameter(layer, "weight", weight_fp8) + replace_parameter(layer, "weight_scale", weight_scale) + layer._already_called_process_weights_after_loading = True diff --git a/vllm_omni/quantization/tools/merge_mxfp8_checkpoint.py b/vllm_omni/quantization/tools/merge_mxfp8_checkpoint.py new file mode 100644 index 000000000..6f13a110c --- /dev/null +++ b/vllm_omni/quantization/tools/merge_mxfp8_checkpoint.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Merge W8A8 MXFP8 quantized Wan2.2 weights into HF Diffusers format for vllm-omni. + +Based on the SGLang convert_wan_to_diffusers approach, extended to: + - Inject `quantization_config` into each transformer's config.json so that + vllm-omni's TransformerConfig.from_dict() auto-detects offline MXFP8 mode + and selects NPUMxfp8LinearMethod (instead of the online/BF16 path). + - Support MoE models (Wan2.2-T2V-A14B / I2V-A14B) with two transformers: + quant_path/high_noise_model -> output/transformer + quant_path/low_noise_model -> output/transformer_2 + +Without the quantization_config injection, FP8 weights are passed to the online +quantization path (npu_dynamic_mx_quant), producing garbage output (visual artifacts). + +Supported model types: + - Wan2.2-T2V-A14B (MoE cascade: transformer + transformer_2) + - Wan2.2-I2V-A14B (MoE cascade: transformer + transformer_2) + - Wan2.2-TI2V-5B (single transformer) + +Usage: + python vllm_omni/quantization/tools/merge_mxfp8_checkpoint.py \\ + --model-type Wan2.2-T2V-A14B \\ + --original-model /path/to/Wan2.2-T2V-A14B-Diffusers \\ + --quant-path /path/to/quant-output \\ + --output-path /path/to/merged-Wan2.2-T2V-A14B-MXFP8 +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import shutil +from typing import Any + +from safetensors.torch import load_file, save_file + +# --------------------------------------------------------------------------- +# Name mapping: quantization tool naming → diffusers naming +# --------------------------------------------------------------------------- + +TRANSFORMER_KEYS_RENAME_DICT: dict[str, str] = { + "time_embedding.0": "condition_embedder.time_embedder.linear_1", + "time_embedding.2": "condition_embedder.time_embedder.linear_2", + "text_embedding.0": "condition_embedder.text_embedder.linear_1", + "text_embedding.2": "condition_embedder.text_embedder.linear_2", + "time_projection.1": "condition_embedder.time_proj", + "head.modulation": "scale_shift_table", + "head.head": "proj_out", + "modulation": "scale_shift_table", + "ffn.0": "ffn.net.0.proj", + "ffn.2": "ffn.net.2", + # Norm order swap (original: norm1, norm3, norm2 → diffusers: norm1, norm2, norm3) + "norm2": "norm__placeholder", + "norm3": "norm2", + "norm__placeholder": "norm3", + # Self-attention + "self_attn.q": "attn1.to_q", + "self_attn.k": "attn1.to_k", + "self_attn.v": "attn1.to_v", + "self_attn.o": "attn1.to_out.0", + "self_attn.norm_q": "attn1.norm_q", + "self_attn.norm_k": "attn1.norm_k", + # Cross-attention + "cross_attn.q": "attn2.to_q", + "cross_attn.k": "attn2.to_k", + "cross_attn.v": "attn2.to_v", + "cross_attn.o": "attn2.to_out.0", + "cross_attn.norm_q": "attn2.norm_q", + "cross_attn.norm_k": "attn2.norm_k", + "attn2.to_k_img": "attn2.add_k_proj", + "attn2.to_v_img": "attn2.add_v_proj", + "attn2.norm_k_img": "attn2.norm_added_k", + # I2V image embedder + "img_emb.proj.0": "condition_embedder.image_embedder.norm1", + "img_emb.proj.1": "condition_embedder.image_embedder.ff.net.0.proj", + "img_emb.proj.3": "condition_embedder.image_embedder.ff.net.2", + "img_emb.proj.4": "condition_embedder.image_embedder.norm2", + "img_emb.emb_pos": "condition_embedder.image_embedder.pos_embed", +} + +SUPPORTED_MODEL_TYPES = ["Wan2.2-T2V-A14B", "Wan2.2-I2V-A14B", "Wan2.2-TI2V-5B"] + +# MoE cascade models have high_noise_model + low_noise_model +CASCADE_MODEL_TYPES = {"Wan2.2-T2V-A14B", "Wan2.2-I2V-A14B"} + +# quantization_config injected into each transformer's config.json. +# is_checkpoint_mxfp8_serialized=True triggers NPUMxfp8LinearMethod (offline path) +# instead of NPUMxfp8OnlineLinearMethod (online/BF16 path). +MXFP8_QUANT_CONFIG: dict[str, Any] = { + "quant_method": "mxfp8", + "is_checkpoint_mxfp8_serialized": True, +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _get_transformer_dirs(model_type: str) -> list[str]: + if model_type in CASCADE_MODEL_TYPES: + return ["transformer", "transformer_2"] + return ["transformer"] + + +def _get_quant_subdir(model_type: str, quant_path: pathlib.Path, transformer_dir: str) -> pathlib.Path: + if model_type in CASCADE_MODEL_TYPES: + sub = "high_noise_model" if transformer_dir == "transformer" else "low_noise_model" + return quant_path / sub + return quant_path + + +def _remap_keys(state_dict: dict, quant_meta: dict) -> tuple[dict, dict]: + """Apply TRANSFORMER_KEYS_RENAME_DICT to both state_dict and quant_meta.""" + new_state: dict = {} + new_meta: dict = {} + + for key, tensor in state_dict.items(): + new_key = key + for src, dst in TRANSFORMER_KEYS_RENAME_DICT.items(): + new_key = new_key.replace(src, dst) + new_state[new_key] = tensor + if key in quant_meta: + new_meta[new_key] = quant_meta[key] + + return new_state, new_meta + + +def _load_safetensors(directory: pathlib.Path, glob: str = "*.safetensors") -> dict: + candidates = sorted(directory.glob(glob)) + if not candidates: + raise FileNotFoundError(f"No safetensors matching '{glob}' found in {directory}") + state_dict = {} + for f in candidates: + state_dict.update(load_file(str(f))) + return state_dict + + +def _load_quant_safetensors(directory: pathlib.Path) -> dict: + try: + return _load_safetensors(directory, "quant_model_weight*.safetensors") + except FileNotFoundError: + return _load_safetensors(directory, "*.safetensors") + + +def _load_quant_meta(directory: pathlib.Path) -> dict: + """Load quant_model_description*.json (maps tensor name → quant type).""" + candidates = sorted(directory.glob("quant_model_description*.json")) + if not candidates: + print(f" WARNING: No quant_model_description*.json in {directory}; treating all as FLOAT") + return {} + with open(candidates[0]) as f: + return json.load(f) + + +# --------------------------------------------------------------------------- +# Per-transformer conversion +# --------------------------------------------------------------------------- + + +def _convert_transformer( + model_type: str, + quant_subdir: pathlib.Path, + output_dir: pathlib.Path, + original_transformer_dir: pathlib.Path, +) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + + # Load original BF16 weights as the merge base so that tensors msModelSlim + # omits (biases, norms, embeddings) are preserved without gaps. + print(f" Loading original BF16 weights from {original_transformer_dir} …") + base_state_dict = _load_safetensors(original_transformer_dir) + print(f" {len(base_state_dict)} BF16 tensors loaded") + + print(f" Loading quantized weights from {quant_subdir} …") + quant_state_dict = _load_quant_safetensors(quant_subdir) + quant_meta = _load_quant_meta(quant_subdir) + print(f" {len(quant_state_dict)} quant tensors, {len(quant_meta)} meta entries") + + # Rename keys from msModelSlim convention to diffusers convention + quant_state_dict, quant_meta = _remap_keys(quant_state_dict, quant_meta) + + # Overlay: start from BF16 base, then apply all msModelSlim tensors on top. + # Quantized weight tensors + their scale tensors override the BF16 originals. + # Non-quantized tensors that msModelSlim emits (FLOAT in quant_meta) keep the + # msModelSlim copy; any tensor msModelSlim omits is retained from the BF16 base. + merged = {**base_state_dict, **quant_state_dict} + + # Save merged safetensors + out_weights = output_dir / "diffusion_pytorch_model.safetensors" + save_file(merged, str(out_weights)) + print(f" Saved {len(merged)} tensors → {out_weights}") + + # Save renamed quant metadata (optional, for reference) + out_meta = output_dir / "quant_model_description.json" + with open(out_meta, "w") as f: + json.dump(quant_meta, f, indent=2) + + # Copy config.json from original transformer and inject quantization_config. + # This is the critical step: TransformerConfig.from_dict() reads + # config["quantization_config"]["quant_method"] to select the quant method, + # and "is_checkpoint_mxfp8_serialized": true to select NPUMxfp8LinearMethod + # (offline path) instead of NPUMxfp8OnlineLinearMethod (online/BF16 path). + src_config = original_transformer_dir / "config.json" + if src_config.is_file(): + with open(src_config) as f: + config = json.load(f) + config["quantization_config"] = MXFP8_QUANT_CONFIG + with open(output_dir / "config.json", "w") as f: + json.dump(config, f, indent=2) + print(f" Injected quantization_config into config.json → {output_dir / 'config.json'}") + else: + print(f" WARNING: No config.json found at {src_config}") + + +# --------------------------------------------------------------------------- +# Main repack +# --------------------------------------------------------------------------- + + +def repack( + model_type: str, + original_model_path: pathlib.Path, + quant_path: pathlib.Path, + output_path: pathlib.Path, +) -> None: + transformer_dirs = _get_transformer_dirs(model_type) + + # Step 1: Copy original model, skipping transformer dirs (will be replaced). + print(f"Copying original model to {output_path} (skipping {transformer_dirs}) …") + shutil.copytree( + str(original_model_path), + str(output_path), + ignore=shutil.ignore_patterns(*transformer_dirs), + ) + + # Step 2+: Convert each transformer. + for tdir in transformer_dirs: + q_subdir = _get_quant_subdir(model_type, quant_path, tdir) + out_tdir = output_path / tdir + orig_tdir = original_model_path / tdir + print(f"\nConverting {tdir} (quant source: {q_subdir.name}) …") + _convert_transformer(model_type, q_subdir, out_tdir, orig_tdir) + + print(f"\nDone. Merged model → {output_path}") + print("\nRun inference (no --quantization flag needed; auto-detected from config.json):") + print(f" python text_to_video.py --model {output_path}") + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--model-type", required=True, choices=SUPPORTED_MODEL_TYPES) + parser.add_argument("--original-model", required=True, help="Original HF Diffusers model directory") + parser.add_argument("--quant-path", required=True, help="msmodelslim quantized weights directory") + parser.add_argument("--output-path", required=True, help="Output directory for merged model") + args = parser.parse_args() + + repack( + model_type=args.model_type, + original_model_path=pathlib.Path(args.original_model), + quant_path=pathlib.Path(args.quant_path), + output_path=pathlib.Path(args.output_path), + ) + + +if __name__ == "__main__": + main()