[Perf][Diffusion] support Sensenova online fp8 (#7955)

Signed-off-by: Dmaner <2663515256@qq.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>

tests/diffusion/models/sensenova_u1/test_sensenova_u1_quantization.py

@@ -0,0 +1,125 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+import torch
+from torch import nn
+
+import vllm_omni.diffusion.models.sensenova_u1.pipeline_sensenova_u1 as pipeline_module
+from vllm_omni.quantization import build_quant_config
+from vllm_omni.quantization.component_config import ComponentQuantizationConfig
+
+pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
+
+
+@pytest.fixture
+def pipeline_setup(monkeypatch):
+    model_config = SimpleNamespace(
+        llm_config=SimpleNamespace(hidden_size=8),
+        vision_config=SimpleNamespace(patch_size=2),
+        downsample_ratio=0.5,
+        use_pixel_head=True,
+        add_noise_scale_embedding=False,
+    )
+    language_model = nn.Module()
+    language_model.model = nn.Module()
+    language_model_class = Mock(return_value=language_model)
+
+    monkeypatch.setattr(pipeline_module, "get_local_device", lambda: torch.device("cpu"))
+    monkeypatch.setattr(pipeline_module, "_resolve_model_path", lambda path: path)
+    monkeypatch.setattr(pipeline_module.SenseNovaU1Config, "from_pretrained", Mock(return_value=model_config))
+    monkeypatch.setattr(pipeline_module.AutoTokenizer, "from_pretrained", Mock(return_value=Mock()))
+    monkeypatch.setattr(pipeline_module, "SenseNovaU1ForCausalLM", language_model_class)
+    monkeypatch.setattr(pipeline_module, "NEOVisionModel", Mock(return_value=nn.Identity()))
+    monkeypatch.setattr(pipeline_module, "ConvDecoder", lambda hidden_size: nn.Identity())
+
+    return model_config, language_model_class
+
+
+@pytest.mark.parametrize(
+    ("quantization", "lora_backend", "lora_path"),
+    [
+        pytest.param(None, "distill", "adapter", id="bf16_lora"),
+        pytest.param("fp8", "distill", None, id="fp8_without_lora"),
+        pytest.param("fp8", "distill", "", id="fp8_empty_lora_path"),
+        pytest.param("component_match", "distill", None, id="component_match"),
+        pytest.param("component_unmatched", "distill", "adapter", id="component_unmatched"),
+        pytest.param("component_default", "distill", None, id="component_default"),
+        pytest.param("component_disabled", "distill", "adapter", id="component_disabled"),
+        pytest.param("fp8", "peft", "adapter", id="peft_not_rejected"),
+        pytest.param("serialized_fp8", "distill", "adapter", id="serialized_not_rejected"),
+    ],
+)
+def test_pipeline_routes_online_fp8_config(pipeline_setup, quantization, lora_backend, lora_path):
+    model_config, language_model_class = pipeline_setup
+    fp8_config = build_quant_config("fp8")
+    serialized_fp8_config = build_quant_config({"method": "fp8", "is_checkpoint_fp8_serialized": True})
+    quant_config, expected_config = {
+        None: (None, None),
+        "fp8": (fp8_config, fp8_config),
+        "serialized_fp8": (serialized_fp8_config, serialized_fp8_config),
+        "component_match": (ComponentQuantizationConfig({"language_model": fp8_config}), fp8_config),
+        "component_unmatched": (ComponentQuantizationConfig({"vision_model": fp8_config}), None),
+        "component_default": (
+            ComponentQuantizationConfig({"vision_model": None}, default_config=fp8_config),
+            fp8_config,
+        ),
+        "component_disabled": (ComponentQuantizationConfig({"language_model": None}, default_config=fp8_config), None),
+    }[quantization]
+    od_config = SimpleNamespace(
+        model="sensenova-test-model",
+        dtype=torch.bfloat16,
+        quantization_config=quant_config,
+        revision=None,
+        lora_backend=lora_backend,
+        lora_path=lora_path,
+        enable_diffusion_pipeline_profiler=False,
+    )
+
+    pipeline_module.SenseNovaU1Pipeline(od_config=od_config)
+
+    language_model_class.assert_called_once_with(
+        model_config.llm_config,
+        quant_config=expected_config,
+        prefix="language_model",
+    )
+
+
+@pytest.mark.parametrize("quantization", ["fp8", "component_match", "component_default"])
+def test_pipeline_rejects_online_fp8_with_distilled_lora_before_loading(pipeline_setup, quantization):
+    _, language_model_class = pipeline_setup
+    fp8_config = build_quant_config("fp8")
+    quant_config = {
+        "fp8": fp8_config,
+        "component_match": ComponentQuantizationConfig({"language_model": fp8_config}),
+        "component_default": ComponentQuantizationConfig({"vision_model": None}, default_config=fp8_config),
+    }[quantization]
+    od_config = SimpleNamespace(
+        model="sensenova-test-model",
+        quantization_config=quant_config,
+        lora_backend="distill",
+        lora_path="adapter",
+    )
+
+    with pytest.raises(ValueError) as exc_info:
+        pipeline_module.SenseNovaU1Pipeline(od_config=od_config)
+
+    assert str(exc_info.value) == (
+        "SenseNova does not support online FP8 with distilled LoRA "
+        "Use BF16 without quantization for distilled LoRA, or omit the LoRA options for online FP8."
+    )
+    language_model_class.assert_not_called()
+    pipeline_module.NEOVisionModel.assert_not_called()
+
+
+def test_distilled_lora_check_does_not_reject_other_quantization():
+    pipeline = pipeline_module.SenseNovaU1Pipeline.__new__(pipeline_module.SenseNovaU1Pipeline)
+    nn.Module.__init__(pipeline)
+    pipeline.od_config = SimpleNamespace(lora_backend="distill", lora_path="adapter")
+    quant_config = Mock()
+    quant_config.get_name.return_value = "int8"
+
+    pipeline._unsupported_quant_methods_check(quant_config)

tests/diffusion/quantization/test_quantization_quality.py

@@ -45,7 +45,6 @@ import pytest
 import torch
 from PIL import Image
 
-from tests.e2e.accuracy.helpers import resolve_device_threshold
 from tests.helpers.mark import hardware_marks
 
 _REPO_ROOT = Path(__file__).resolve().parents[3]
@@ -85,6 +84,7 @@ class QualityTestConfig:
     guidance_scale: float | None = None
     sigmas: list[float] | None = None
     enable_cpu_offload: bool = False
+    diffusion_attention_backend: str | None = None
 
     def baseline_ref(self) -> str:
         return self.baseline_model or self.model or ""
@@ -206,6 +206,21 @@ QUALITY_CONFIGS = [
         seed=142,
         num_inference_steps=20,
     ),
+    QualityTestConfig(
+        id="fp8_sensenova_u1",
+        model=os.environ.get("SENSENOVA_U1_MODEL_PATH", "SenseNova/SenseNova-U1.5-8B-MoT"),
+        quantization="fp8",
+        task="t2i",
+        prompt="Close portrait of an elderly woman by a farmhouse window, warm natural light.",
+        # Repository image-gate default, not a SenseNova-calibrated threshold.
+        max_lpips=0.15,
+        height=1024,
+        width=1024,
+        num_inference_steps=50,
+        seed=42,
+        negative_prompt=None,
+        diffusion_attention_backend="TORCH_SDPA",
+    ),
     QualityTestConfig(
         id="fp8_ltx2",
         model="Lightricks/LTX-2",
@@ -276,6 +291,15 @@ def _maybe_save_output(output_dir: Path | None, config: QualityTestConfig, label
 # ---------------------------------------------------------------------------
 
 
+def _build_omni_kwargs(config: QualityTestConfig, model: str) -> dict:
+    kwargs = {"model": model, "enforce_eager": True}
+    if config.enable_cpu_offload:
+        kwargs["enable_cpu_offload"] = True
+    if config.diffusion_attention_backend is not None:
+        kwargs["diffusion_attention_backend"] = config.diffusion_attention_backend
+    return kwargs
+
+
 def _generate_image(omni, config: QualityTestConfig):
     """Generate a single image, return (PIL.Image, peak_mem_gib)."""
     from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@@ -294,6 +318,7 @@ def _generate_image(omni, config: QualityTestConfig):
         OmniDiffusionSamplingParams(
             height=config.height,
             width=config.width,
+            seed=config.seed,
             generator=generator,
             num_inference_steps=config.num_inference_steps,
             guidance_scale=config.guidance_scale,
@@ -534,6 +559,8 @@ _OUTPUT_DIR = Path(os.environ["VLLM_OMNI_QUALITY_OUTPUT_DIR"]) if "VLLM_OMNI_QUA
 
 def _quality_param(c: QualityTestConfig):
     marks = list(_marks)
+    if c.id == "fp8_sensenova_u1":
+        marks = hardware_marks(res={"cuda": "H200"})
     if c.id == "fp8_qwen_image":
         marks.append(
             pytest.mark.skip(reason="Qwen-Image FP8 quality gate temporarily disabled (see CI / issue tracker).")
@@ -549,6 +576,7 @@ def _quality_param(c: QualityTestConfig):
 )
 def test_quantization_quality(config: QualityTestConfig):
     """Validate that quantized output stays within LPIPS threshold of BF16."""
+    from tests.e2e.accuracy.helpers import resolve_device_threshold
     from vllm_omni.entrypoints.omni import Omni
 
     generate_fn = _generate_video if config.task == "t2v" else _generate_image
@@ -560,9 +588,7 @@ def test_quantization_quality(config: QualityTestConfig):
     # to 0.1291 that way in build 2954). Mirrors
     # vllm_omni/quantization/tools/compare_diffusion_trajectory_similarity.py.
     # --- BF16 baseline ---
-    bl_kwargs: dict = {"model": config.baseline_ref(), "enforce_eager": True}
-    if config.enable_cpu_offload:
-        bl_kwargs["enable_cpu_offload"] = True
+    bl_kwargs = _build_omni_kwargs(config, config.baseline_ref())
     omni_bl = Omni(**bl_kwargs)
     baseline_out, bl_mem = generate_fn(omni_bl, config)
     omni_bl.shutdown()
@@ -572,9 +598,7 @@ def test_quantization_quality(config: QualityTestConfig):
 
     # --- Quantized ---
     quantization = config.quantization_ref()
-    qt_kwargs: dict = {"model": config.quantized_ref(), "enforce_eager": True}
-    if config.enable_cpu_offload:
-        qt_kwargs["enable_cpu_offload"] = True
+    qt_kwargs = _build_omni_kwargs(config, config.quantized_ref())
     if quantization is None:
         omni_qt = Omni(**qt_kwargs)
     else: