diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index ef99257fb..a324d6bbf 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -591,6 +591,7 @@ steps: tests/e2e/online_serving/test_longcat_image_edit_expansion.py tests/e2e/online_serving/test_flux_2_dev_expansion.py tests/e2e/online_serving/test_bagel_expansion.py + tests/e2e/offline_inference/test_sensenova_u1_expansion.py -m "full_model and diffusion and H100 and not distributed_cuda" --run-level "full_model" agents: diff --git a/tests/e2e/offline_inference/test_sensenova_u1_expansion.py b/tests/e2e/offline_inference/test_sensenova_u1_expansion.py new file mode 100644 index 000000000..1f9769f81 --- /dev/null +++ b/tests/e2e/offline_inference/test_sensenova_u1_expansion.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +Expanded end-to-end test for SenseNova-U1 in offline mode. + +This test file primarily covers end-to-end tests for LoRA support. + +Validates that LoRA adapters are correctly loaded, applied with controllable +scale, and cleanly deactivated. Uses a synthetic rank-4 adapter targeting the +first decoder layer's gen-path (``_mot_gen``) output projection, so the +perturbation is exercised during the denoising loop. + +Assertions: + (a) LoRA at scale=1.0 visibly changes the output (diff > 0.5) + (b) scale=2.0 produces a different delta than scale=1.0 (scale sensitivity) + (c) The delta is bounded (not corrupted) + (d) Deactivating LoRA exactly restores the baseline (diff == 0) + +Observed on H100 (seed=42, 8 steps, 512x512): diff_1x~62.2, diff_2x~76.9, +diff_restored=0.0. Thresholds match test_bagel_expansion.py and hold with margin. +""" + +import json +import os +from pathlib import Path + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +import numpy as np +import pytest +import torch +from PIL import Image +from safetensors.torch import save_file + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniRunner +from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams +from vllm_omni.lora.request import LoRARequest +from vllm_omni.lora.utils import stable_lora_int_id + +MODEL = "SenseNova/SenseNova-U1-8B-MoT" +DEFAULT_PROMPT = "a photo of a cat sitting on a laptop keyboard" +SIZE = (512, 512) # (width, height); small for fast E2E + +# SenseNova-U1-8B-MoT hidden_size (Qwen3-based). o_proj is square (hidden x hidden) +# regardless of GQA, so this is the robust LoRA target. Verify against config.json. +_LORA_DIM = 4096 +_LORA_MODULE = "language_model.model.layers.0.self_attn.o_proj_mot_gen" +_LORA_RANK = 4 + + +# --------------------------------------------------------------------------- +# Helpers (reused from test_sensenova_u1_text2img.py patterns) +# --------------------------------------------------------------------------- + + +def _build_sampling_params( + lora_request: LoRARequest | None = None, + lora_scale: float = 1.0, +) -> OmniDiffusionSamplingParams: + return OmniDiffusionSamplingParams( + height=SIZE[1], + width=SIZE[0], + seed=42, + num_inference_steps=8, + extra_args={ + "cfg_scale": 4.0, + "cfg_norm": "none", + "timestep_shift": 3.0, + "cfg_interval": (0.0, 1.0), + "batch_size": 1, + "think": False, + "t_eps": 0.02, + }, + lora_request=lora_request, + lora_scale=lora_scale, + ) + + +def _extract_generated_image(omni_outputs: list) -> Image.Image | None: + for req_output in omni_outputs: + if images := getattr(req_output, "images", None): + return images[0] + return None + + +def _generate(omni: Omni, lora_request: LoRARequest | None = None, lora_scale: float = 1.0) -> Image.Image: + outputs = list( + omni.generate( + prompts={"prompt": DEFAULT_PROMPT, "modalities": ["image"]}, + sampling_params_list=_build_sampling_params(lora_request, lora_scale), + ) + ) + img = _extract_generated_image(outputs) + assert img is not None, "No image generated" + return img + + +def _make_file_lora_request(adapter_dir: Path) -> LoRARequest: + """Write synthetic adapter to disk and return a file-backed LoRARequest.""" + adapter_dir.mkdir(parents=True, exist_ok=True) + gen = torch.Generator().manual_seed(42) + lora_a = torch.randn((_LORA_RANK, _LORA_DIM), dtype=torch.float32, generator=gen) * 0.1 + lora_b = torch.randn((_LORA_DIM, _LORA_RANK), dtype=torch.float32, generator=gen) * 0.5 + save_file( + { + f"base_model.model.{_LORA_MODULE}.lora_A.weight": lora_a, + f"base_model.model.{_LORA_MODULE}.lora_B.weight": lora_b, + }, + str(adapter_dir / "adapter_model.safetensors"), + ) + (adapter_dir / "adapter_config.json").write_text( + json.dumps({"r": _LORA_RANK, "lora_alpha": _LORA_RANK, "target_modules": [_LORA_MODULE]}), + encoding="utf-8", + ) + lora_dir = str(adapter_dir) + return LoRARequest(lora_name="test_file", lora_int_id=stable_lora_int_id(lora_dir), lora_path=lora_dir) + + +# --------------------------------------------------------------------------- +# Test +# --------------------------------------------------------------------------- + + +@pytest.mark.full_model +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}) +def test_sensenova_u1_lora_scale_and_deactivation(tmp_path) -> None: + """Validate LoRA effect, scale sensitivity, bounded perturbation, and clean deactivation.""" + lora_request = _make_file_lora_request(tmp_path / "sensenova_lora") + + with OmniRunner(MODEL, stage_configs_path=None) as runner: + omni = runner.omni + baseline = _generate(omni) + img_1x = _generate(omni, lora_request, lora_scale=1.0) + img_2x = _generate(omni, lora_request, lora_scale=2.0) + restored = _generate(omni) + + baseline_arr = np.array(baseline, dtype=np.int16) + diff_1x = np.abs(baseline_arr - np.array(img_1x, dtype=np.int16)).mean() + diff_2x = np.abs(baseline_arr - np.array(img_2x, dtype=np.int16)).mean() + diff_restored = np.abs(baseline_arr - np.array(restored, dtype=np.int16)).mean() + + # (a) Adapter has visible effect at both scales + assert diff_1x > 0.5, f"LoRA scale=1.0 had no visible effect: diff={diff_1x}" + assert diff_2x > 0.5, f"LoRA scale=2.0 had no visible effect: diff={diff_2x}" + + # (b) Different scales produce different outputs + assert not np.isclose(diff_1x, diff_2x, atol=1.0), ( + f"LoRA scale has no effect: diff_1x={diff_1x:.2f}, diff_2x={diff_2x:.2f}" + ) + + # (c) Output is not corrupted + assert diff_1x < 80, f"LoRA output looks corrupted: diff_1x={diff_1x}" + assert diff_2x < 120, f"LoRA output looks corrupted: diff_2x={diff_2x}" + + # (d) Deactivation fully restores base model + assert diff_restored == 0.0, f"Base model not restored after LoRA deactivation: diff={diff_restored}" diff --git a/vllm_omni/diffusion/lora/manager.py b/vllm_omni/diffusion/lora/manager.py index 63e8d9a96..8ba30b79e 100644 --- a/vllm_omni/diffusion/lora/manager.py +++ b/vllm_omni/diffusion/lora/manager.py @@ -366,7 +366,11 @@ class DiffusionLoRAManager: fully_sharded_loras=False, ) - for component_name in ("transformer", "transformer_2", "dit", "bagel"): + # Default denoising components plus any a pipeline opts into via + # ``_lora_components`` (opt-in; other pipelines unchanged). + default_components = ("transformer", "transformer_2", "dit", "bagel") + extra_components = tuple(getattr(self.pipeline, "_lora_components", ()) or ()) + for component_name in (*default_components, *extra_components): if not hasattr(self.pipeline, component_name): continue component = getattr(self.pipeline, component_name) diff --git a/vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py b/vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py index fe8695b07..fa7f95c66 100644 --- a/vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py +++ b/vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py @@ -536,6 +536,10 @@ class SenseNovaU1Pipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipeli _vae_modules: ClassVar[list[str]] = [] _resident_modules: ClassVar[list[str]] = ["fm_modules"] + # Top-level module(s) the diffusion LoRA manager scans (both MoT branches). + # TODO: promote to a shared LoRA contract/mixin instead of per-pipeline opt-in. + _lora_components: ClassVar[list[str]] = ["language_model"] + def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""): super().__init__() self.od_config = od_config @@ -1454,6 +1458,7 @@ class SenseNovaU1Pipeline(nn.Module, SupportsComponentDiscovery, DiffusionPipeli (".gate_up_proj", ".gate_proj", 0), (".gate_up_proj", ".up_proj", 1), ] + self.stacked_params_mapping = stacked_params_mapping params_dict = dict(self.named_parameters()) loaded_params: set[str] = set()