From 79a46a4f5471e7b8f3ce093d0cf0b94ba7264c04 Mon Sep 17 00:00:00 2001 From: DMAN <32004742+Dmaner@users.noreply.github.com> Date: Mon, 28 Sep 2026 23:17:37 +0800 Subject: [PATCH] [Perf][Diffusion] support Sensenova online fp8 (#7955) Signed-off-by: Dmaner <2663515256@qq.com> Co-authored-by: Hongsheng Liu --- docs/user_guide/diffusion_features.md | 3 +- docs/user_guide/quantization/fp8.md | 1 + recipes/SenseNova/SenseNova-U1.5.md | 37 +++++- .../test_sensenova_u1_quantization.py | 125 ++++++++++++++++++ .../quantization/test_quantization_quality.py | 38 +++++- .../sensenova_u1/pipeline_sensenova_u1.py | 20 +++ 6 files changed, 215 insertions(+), 9 deletions(-) create mode 100644 tests/diffusion/models/sensenova_u1/test_sensenova_u1_quantization.py diff --git a/docs/user_guide/diffusion_features.md b/docs/user_guide/diffusion_features.md index cad0713fc..7a80ac781 100644 --- a/docs/user_guide/diffusion_features.md +++ b/docs/user_guide/diffusion_features.md @@ -163,7 +163,8 @@ The following tables show which models support each feature: | **Qwen-Image-Edit** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ | | **Qwen-Image-Edit-2509** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ (decode) | ✅ | ❌ | ❌ | | **Qwen-Image-Layered** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ | -| **SenseNova-U1 / U1.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | +| **SenseNova-U1** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | +| **SenseNova-U1.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | | **SenseNova-U1-A3B** | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | **Stable-Diffusion-XL** | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ | | **Stable-Diffusion3.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ (decode) | ❌ | ❌ | diff --git a/docs/user_guide/quantization/fp8.md b/docs/user_guide/quantization/fp8.md index 8d0090ec0..9dfe1310d 100644 --- a/docs/user_guide/quantization/fp8.md +++ b/docs/user_guide/quantization/fp8.md @@ -106,6 +106,7 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)]) | HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | | | Cosmos3 | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | Yes | Not validated | All layers | None | | | MiniMax-H3 | `MiniMaxAI/MiniMax-H3` (`FL2VA` / `Ref2VA`) | Yes | Not validated | `quantization="fp8"` quantizes eligible DiT and text-encoder linears; mixed-precision input/output heads stay FP32 | None | ✅︎ | +| SenseNova-U1.5 | `sensenova/SenseNova-U1.5-8B-MoT` | Yes | Not validated | UND/GEN language-model linears only | None | | ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS) diff --git a/recipes/SenseNova/SenseNova-U1.5.md b/recipes/SenseNova/SenseNova-U1.5.md index 80b40ab2a..2a74479de 100644 --- a/recipes/SenseNova/SenseNova-U1.5.md +++ b/recipes/SenseNova/SenseNova-U1.5.md @@ -79,6 +79,9 @@ python examples/offline_inference/text_to_image/text_to_image.py \ classifier-free guidance; the default `4.0` applies guidance twice and produces a blown-out, posterised image. +Online FP8 does not support the 8-step distilled LoRA. +Use BF16 without `--quantization fp8` for this LoRA. + ##### Online serving ```bash @@ -90,7 +93,39 @@ python examples/online_serving/sensenova_u1/openai_chat_client.py \ `-s` takes the base URL; the client appends `/v1` itself. -#### Measured latency (1x A800 80GB, 25 steps, median of 3 after a warmup) +##### Online FP8 quantization + +`--quantization fp8` quantizes only eligible attention and MLP linears in the +`language_model` understanding and generation branches at load time. Other layers +retain their existing precision. + +On the tested A800 (SM80) setup with vLLM 0.29.0, +`VLLM_DISABLED_KERNELS=CutlassFP8ScaledMMLinearKernel` is **required** for online +FP8 to start successfully. Without it, the selected CUTLASS kernel fails during +the startup dummy run with `RuntimeError: cutlass_scaled_mm_sm80_epilogue`. +Disabling this kernel selects Marlin W8A16 (FP8 weights with BF16 activations). + +```bash +VLLM_DISABLED_KERNELS=CutlassFP8ScaledMMLinearKernel \ +vllm serve sensenova/SenseNova-U1.5-8B-MoT --omni \ + --quantization fp8 --port 8091 +``` + +#### Measured BF16 vs FP8 performance (1x A800 80GB, 28 steps, P95 over 10 prompts) + +vLLM 0.29.0, BF16 base dtype, seed 42, TORCH_SDPA, eager execution, TP=1, batch size 1, +paged decode off, no LoRA. Each resolution uses the same 10 prompts for BF16 and FP8. +Both runs set `VLLM_DISABLED_KERNELS=CutlassFP8ScaledMMLinearKernel`. + +| Resolution | Peak reserved memory (BF16 → FP8) | Step latency (BF16 → FP8) | +| --- | --- | --- | +| 512x512 | 33.166 → 18.697 GiB | 63.674 → 81.136 ms | +| 1024x1024 | 33.555 → 18.561 GiB | 187.867 → 229.115 ms | + +Step latency is the P95 of per-request mean step times, excluding warmup requests; FP8 reduced +memory usage but increased step latency. + +#### Measured BF16 latency (1x A800 80GB, 25 steps, median of 3 after a warmup) | Resolution | Step latency | Total | | --- | --- | --- | diff --git a/tests/diffusion/models/sensenova_u1/test_sensenova_u1_quantization.py b/tests/diffusion/models/sensenova_u1/test_sensenova_u1_quantization.py new file mode 100644 index 000000000..9dc990b69 --- /dev/null +++ b/tests/diffusion/models/sensenova_u1/test_sensenova_u1_quantization.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch +from torch import nn + +import vllm_omni.diffusion.models.sensenova_u1.pipeline_sensenova_u1 as pipeline_module +from vllm_omni.quantization import build_quant_config +from vllm_omni.quantization.component_config import ComponentQuantizationConfig + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + + +@pytest.fixture +def pipeline_setup(monkeypatch): + model_config = SimpleNamespace( + llm_config=SimpleNamespace(hidden_size=8), + vision_config=SimpleNamespace(patch_size=2), + downsample_ratio=0.5, + use_pixel_head=True, + add_noise_scale_embedding=False, + ) + language_model = nn.Module() + language_model.model = nn.Module() + language_model_class = Mock(return_value=language_model) + + monkeypatch.setattr(pipeline_module, "get_local_device", lambda: torch.device("cpu")) + monkeypatch.setattr(pipeline_module, "_resolve_model_path", lambda path: path) + monkeypatch.setattr(pipeline_module.SenseNovaU1Config, "from_pretrained", Mock(return_value=model_config)) + monkeypatch.setattr(pipeline_module.AutoTokenizer, "from_pretrained", Mock(return_value=Mock())) + monkeypatch.setattr(pipeline_module, "SenseNovaU1ForCausalLM", language_model_class) + monkeypatch.setattr(pipeline_module, "NEOVisionModel", Mock(return_value=nn.Identity())) + monkeypatch.setattr(pipeline_module, "ConvDecoder", lambda hidden_size: nn.Identity()) + + return model_config, language_model_class + + +@pytest.mark.parametrize( + ("quantization", "lora_backend", "lora_path"), + [ + pytest.param(None, "distill", "adapter", id="bf16_lora"), + pytest.param("fp8", "distill", None, id="fp8_without_lora"), + pytest.param("fp8", "distill", "", id="fp8_empty_lora_path"), + pytest.param("component_match", "distill", None, id="component_match"), + pytest.param("component_unmatched", "distill", "adapter", id="component_unmatched"), + pytest.param("component_default", "distill", None, id="component_default"), + pytest.param("component_disabled", "distill", "adapter", id="component_disabled"), + pytest.param("fp8", "peft", "adapter", id="peft_not_rejected"), + pytest.param("serialized_fp8", "distill", "adapter", id="serialized_not_rejected"), + ], +) +def test_pipeline_routes_online_fp8_config(pipeline_setup, quantization, lora_backend, lora_path): + model_config, language_model_class = pipeline_setup + fp8_config = build_quant_config("fp8") + serialized_fp8_config = build_quant_config({"method": "fp8", "is_checkpoint_fp8_serialized": True}) + quant_config, expected_config = { + None: (None, None), + "fp8": (fp8_config, fp8_config), + "serialized_fp8": (serialized_fp8_config, serialized_fp8_config), + "component_match": (ComponentQuantizationConfig({"language_model": fp8_config}), fp8_config), + "component_unmatched": (ComponentQuantizationConfig({"vision_model": fp8_config}), None), + "component_default": ( + ComponentQuantizationConfig({"vision_model": None}, default_config=fp8_config), + fp8_config, + ), + "component_disabled": (ComponentQuantizationConfig({"language_model": None}, default_config=fp8_config), None), + }[quantization] + od_config = SimpleNamespace( + model="sensenova-test-model", + dtype=torch.bfloat16, + quantization_config=quant_config, + revision=None, + lora_backend=lora_backend, + lora_path=lora_path, + enable_diffusion_pipeline_profiler=False, + ) + + pipeline_module.SenseNovaU1Pipeline(od_config=od_config) + + language_model_class.assert_called_once_with( + model_config.llm_config, + quant_config=expected_config, + prefix="language_model", + ) + + +@pytest.mark.parametrize("quantization", ["fp8", "component_match", "component_default"]) +def test_pipeline_rejects_online_fp8_with_distilled_lora_before_loading(pipeline_setup, quantization): + _, language_model_class = pipeline_setup + fp8_config = build_quant_config("fp8") + quant_config = { + "fp8": fp8_config, + "component_match": ComponentQuantizationConfig({"language_model": fp8_config}), + "component_default": ComponentQuantizationConfig({"vision_model": None}, default_config=fp8_config), + }[quantization] + od_config = SimpleNamespace( + model="sensenova-test-model", + quantization_config=quant_config, + lora_backend="distill", + lora_path="adapter", + ) + + with pytest.raises(ValueError) as exc_info: + pipeline_module.SenseNovaU1Pipeline(od_config=od_config) + + assert str(exc_info.value) == ( + "SenseNova does not support online FP8 with distilled LoRA " + "Use BF16 without quantization for distilled LoRA, or omit the LoRA options for online FP8." + ) + language_model_class.assert_not_called() + pipeline_module.NEOVisionModel.assert_not_called() + + +def test_distilled_lora_check_does_not_reject_other_quantization(): + pipeline = pipeline_module.SenseNovaU1Pipeline.__new__(pipeline_module.SenseNovaU1Pipeline) + nn.Module.__init__(pipeline) + pipeline.od_config = SimpleNamespace(lora_backend="distill", lora_path="adapter") + quant_config = Mock() + quant_config.get_name.return_value = "int8" + + pipeline._unsupported_quant_methods_check(quant_config) diff --git a/tests/diffusion/quantization/test_quantization_quality.py b/tests/diffusion/quantization/test_quantization_quality.py index 4bace0da5..2b9bf38d0 100644 --- a/tests/diffusion/quantization/test_quantization_quality.py +++ b/tests/diffusion/quantization/test_quantization_quality.py @@ -45,7 +45,6 @@ import pytest import torch from PIL import Image -from tests.e2e.accuracy.helpers import resolve_device_threshold from tests.helpers.mark import hardware_marks _REPO_ROOT = Path(__file__).resolve().parents[3] @@ -85,6 +84,7 @@ class QualityTestConfig: guidance_scale: float | None = None sigmas: list[float] | None = None enable_cpu_offload: bool = False + diffusion_attention_backend: str | None = None def baseline_ref(self) -> str: return self.baseline_model or self.model or "" @@ -206,6 +206,21 @@ QUALITY_CONFIGS = [ seed=142, num_inference_steps=20, ), + QualityTestConfig( + id="fp8_sensenova_u1", + model=os.environ.get("SENSENOVA_U1_MODEL_PATH", "SenseNova/SenseNova-U1.5-8B-MoT"), + quantization="fp8", + task="t2i", + prompt="Close portrait of an elderly woman by a farmhouse window, warm natural light.", + # Repository image-gate default, not a SenseNova-calibrated threshold. + max_lpips=0.15, + height=1024, + width=1024, + num_inference_steps=50, + seed=42, + negative_prompt=None, + diffusion_attention_backend="TORCH_SDPA", + ), QualityTestConfig( id="fp8_ltx2", model="Lightricks/LTX-2", @@ -276,6 +291,15 @@ def _maybe_save_output(output_dir: Path | None, config: QualityTestConfig, label # --------------------------------------------------------------------------- +def _build_omni_kwargs(config: QualityTestConfig, model: str) -> dict: + kwargs = {"model": model, "enforce_eager": True} + if config.enable_cpu_offload: + kwargs["enable_cpu_offload"] = True + if config.diffusion_attention_backend is not None: + kwargs["diffusion_attention_backend"] = config.diffusion_attention_backend + return kwargs + + def _generate_image(omni, config: QualityTestConfig): """Generate a single image, return (PIL.Image, peak_mem_gib).""" from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -294,6 +318,7 @@ def _generate_image(omni, config: QualityTestConfig): OmniDiffusionSamplingParams( height=config.height, width=config.width, + seed=config.seed, generator=generator, num_inference_steps=config.num_inference_steps, guidance_scale=config.guidance_scale, @@ -534,6 +559,8 @@ _OUTPUT_DIR = Path(os.environ["VLLM_OMNI_QUALITY_OUTPUT_DIR"]) if "VLLM_OMNI_QUA def _quality_param(c: QualityTestConfig): marks = list(_marks) + if c.id == "fp8_sensenova_u1": + marks = hardware_marks(res={"cuda": "H200"}) if c.id == "fp8_qwen_image": marks.append( pytest.mark.skip(reason="Qwen-Image FP8 quality gate temporarily disabled (see CI / issue tracker).") @@ -549,6 +576,7 @@ def _quality_param(c: QualityTestConfig): ) def test_quantization_quality(config: QualityTestConfig): """Validate that quantized output stays within LPIPS threshold of BF16.""" + from tests.e2e.accuracy.helpers import resolve_device_threshold from vllm_omni.entrypoints.omni import Omni generate_fn = _generate_video if config.task == "t2v" else _generate_image @@ -560,9 +588,7 @@ def test_quantization_quality(config: QualityTestConfig): # to 0.1291 that way in build 2954). Mirrors # vllm_omni/quantization/tools/compare_diffusion_trajectory_similarity.py. # --- BF16 baseline --- - bl_kwargs: dict = {"model": config.baseline_ref(), "enforce_eager": True} - if config.enable_cpu_offload: - bl_kwargs["enable_cpu_offload"] = True + bl_kwargs = _build_omni_kwargs(config, config.baseline_ref()) omni_bl = Omni(**bl_kwargs) baseline_out, bl_mem = generate_fn(omni_bl, config) omni_bl.shutdown() @@ -572,9 +598,7 @@ def test_quantization_quality(config: QualityTestConfig): # --- Quantized --- quantization = config.quantization_ref() - qt_kwargs: dict = {"model": config.quantized_ref(), "enforce_eager": True} - if config.enable_cpu_offload: - qt_kwargs["enable_cpu_offload"] = True + qt_kwargs = _build_omni_kwargs(config, config.quantized_ref()) if quantization is None: omni_qt = Omni(**qt_kwargs) else: diff --git a/vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py b/vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py index 79d42d455..8595fb8d5 100644 --- a/vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py +++ b/vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py @@ -30,6 +30,7 @@ import torchvision.transforms as T from PIL import Image from transformers import AutoTokenizer from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin @@ -47,6 +48,7 @@ from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch +from vllm_omni.quantization import resolve_component_quant_config from vllm_omni.transformers_utils.configs.sensenova_u1 import ( SenseNovaU1Config, ) @@ -498,7 +500,9 @@ class SenseNovaU1Pipeline( def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): super().__init__() + quant_config = resolve_component_quant_config(od_config.quantization_config, "language_model") self.od_config = od_config + self._unsupported_quant_methods_check(quant_config) self.device = get_local_device() model_path = od_config.model self.local_model_path = _resolve_model_path(model_path) @@ -514,6 +518,7 @@ class SenseNovaU1Pipeline( # Language model (TP-aware) self.language_model = SenseNovaU1ForCausalLM( self.llm_cfg, + quant_config=quant_config, prefix="language_model", ) # Cache-DiT hooks pipeline.transformer(.blocks), so it must point at the @@ -591,6 +596,21 @@ class SenseNovaU1Pipeline( # Helpers # ----------------------------------------------------------------------- + def _unsupported_quant_methods_check(self, quant_config: QuantizationConfig | None) -> None: + """Reject online FP8 combined with distilled LoRA.""" + if ( + quant_config is None + or quant_config.get_name() != "fp8" + or getattr(quant_config, "is_checkpoint_fp8_serialized", False) + ): + return + + if self.od_config.lora_backend == "distill" and self.od_config.lora_path: + raise ValueError( + "SenseNova does not support online FP8 with distilled LoRA " + "Use BF16 without quantization for distilled LoRA, or omit the LoRA options for online FP8." + ) + def _extract_feature(self, pixel_values, gen_model=False, grid_hw=None): if gen_model: return self.fm_modules["vision_model_mot_gen"](pixel_values=pixel_values, grid_hw=grid_hw)