[Model] Add Krea 2 text-to-image diffusion model (#4730)

Signed-off-by: Abhinay1997 <devarintinagasaiabhinay@gmail.com>
This commit is contained in:
Naga Sai Abhinay Devarinti authored and GitHub committed 2026-07-11 16:25:41 -07:00
1 parent b95905deb1
commit 6581e2afa9
14 files changed
+1666

No files matched your search

@@ -0,0 +1,155 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""L4 functionality tests for the Krea 2 text-to-image diffusion pipeline (``Krea2Pipeline``).
Runs against the public few-step distilled checkpoint ``krea/Krea-2-Turbo`` by default (override with the
``KREA2_MODEL`` environment variable, e.g. ``krea/Krea-2-Raw`` for the Raw checkpoint or a local diffusers
directory). They cover a basic functional smoke plus the layerwise-CPU-offload path that the pipeline declares
via ``SupportsComponentDiscovery`` / ``_layerwise_offload_blocks_attrs``.
"""
import os
import numpy as np
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunnerHandler
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.lora.request import LoRARequest
from vllm_omni.lora.utils import stable_lora_int_id
MODEL = os.environ.get("KREA2_MODEL", "krea/Krea-2-Turbo")
# vLLM-Omni-compatible PEFT repackaging of krea/Krea-2-LoRA-darkbrush (264 modules, r=alpha=32).
LORA = os.environ.get("KREA2_LORA", "NagaSaiAbhinay/Krea-2-vllm-darkbrush-LoRA")
PROMPT = "a fox in the snow, photorealistic"
# darkbrush is a trigger-word LoRA: the trained monochrome ink-wash style only manifests when
# its trigger phrase is present, so the LoRA test prompt appends it for a strong, stable signal.
LORA_TRIGGER = "monochrome ink wash style"
LORA_PROMPT = f"a fox in the snow, {LORA_TRIGGER}"
pytestmark = [
pytest.mark.diffusion,
pytest.mark.full_model,
]
def _sampling() -> OmniDiffusionSamplingParams:
# Small resolution + few steps to keep the L4 case light. guidance_scale resolution is checkpoint-aware inside
# the pipeline (distilled -> no-CFG, Raw -> CFG), so this stays agnostic to which checkpoint KREA2_MODEL points at.
return OmniDiffusionSamplingParams(
height=512,
width=512,
num_inference_steps=8,
guidance_scale=0.0,
seed=42,
)
@hardware_test(res={"cuda": "H100"})
@pytest.mark.parametrize("omni_runner", [(MODEL, None)], indirect=True)
def test_krea2_text_to_image_001(omni_runner_handler: OmniRunnerHandler) -> None:
"""Basic functional smoke: a single prompt produces a decoded image."""
omni_runner_handler.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
@hardware_test(res={"cuda": "H100"})
@pytest.mark.parametrize(
"omni_runner",
[(MODEL, None, {"enable_layerwise_offload": True})],
indirect=True,
)
def test_krea2_layerwise_offload(omni_runner_handler: OmniRunnerHandler) -> None:
"""Exercise layerwise CPU offload on the DiT (SupportsComponentDiscovery + _layerwise_offload_blocks_attrs)."""
omni_runner_handler.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
@hardware_test(res={"cuda": "H100"})
@pytest.mark.cache
@pytest.mark.parametrize("omni_runner", [(MODEL, None, {"cache_backend": "cache_dit"})], indirect=True)
def test_krea2_cache_dit(omni_runner_handler: OmniRunnerHandler) -> None:
"""Exercise Cache-DiT on Krea 2 via the custom Krea2Pipeline enabler (``enable_cache_for_krea2``).
Validates the docs' Cache-DiT support claim for Krea 2. ``has_separate_cfg`` is checkpoint-aware
(False for the distilled Turbo no-CFG path, True for the Raw CFG path); the default Turbo checkpoint
exercises the no-CFG branch.
"""
omni_runner_handler.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize(
"omni_runner",
# Standalone HSDP: use_hsdp=True with an explicit hsdp_shard_size shards the DiT weights
# across 2 ranks (replicate_size defaults to 1 -> world_size = 1 * 2 = 2). HSDP must keep
# tensor_parallel_size/data_parallel_size at 1 (see vllm_omni/diffusion/data.py __post_init__).
# Also validated locally at 8 GPUs (hsdp_shard_size=8); CI shard provisions 2.
[(MODEL, None, {"use_hsdp": True, "hsdp_shard_size": 2})],
indirect=True,
)
def test_krea2_hsdp(omni_runner_handler: OmniRunnerHandler) -> None:
"""Exercise Hybrid Sharded Data Parallel (HSDP) weight sharding on the DiT across 2 GPUs.
Validates the docs' HSDP-support claim for Krea 2. Scheduled on the multi-GPU (distributed_cuda)
shard by ``num_cards=2``; skipped automatically on boxes with fewer than 2 CUDA devices.
"""
omni_runner_handler.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
@hardware_test(res={"cuda": "H100"}, num_cards=2)
@pytest.mark.parametrize(
"omni_runner",
# VAE patch-parallel decode: partition the VAE decode across the TP world. Mirrors the online
# "tp_vae_patch" case (tensor_parallel_size=2 + vae_patch_parallel_size=2) from the wan2.2/z-image
# expansion suites, so vae_patch_parallel_size must equal the world size established by TP.
# Also validated locally at 8 GPUs (tp=8, vae_patch=8); CI shard provisions 2.
[(MODEL, None, {"tensor_parallel_size": 2, "vae_patch_parallel_size": 2})],
indirect=True,
)
def test_krea2_vae_patch_parallel(omni_runner_handler: OmniRunnerHandler) -> None:
"""Exercise VAE patch-parallel (decode) across 2 GPUs.
Validates the docs' VAE-patch-parallel (decode) support claim for Krea 2. Scheduled on the
multi-GPU (distributed_cuda) shard by ``num_cards=2``; skipped automatically on boxes with
fewer than 2 CUDA devices.
"""
omni_runner_handler.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
def _generate(handler: OmniRunnerHandler, lora_request: LoRARequest | None, lora_scale: float = 1.0) -> np.ndarray:
sp = _sampling()
sp.lora_request = lora_request
sp.lora_scale = lora_scale
resp = handler.send_diffusion_request({"model": MODEL, "prompt": LORA_PROMPT, "sampling_params": sp})
return np.asarray(resp.images[0].convert("RGB"), dtype=np.int16)
@hardware_test(res={"cuda": "H100"})
@pytest.mark.parametrize("omni_runner", [(MODEL, None)], indirect=True)
def test_krea2_lora(omni_runner_handler: OmniRunnerHandler) -> None:
"""Validate diffusion LoRA on Krea 2: visible effect, scale sensitivity, clean deactivation.
Uses ``LORA`` (a vLLM-Omni-compatible PEFT repackaging of ``krea/Krea-2-LoRA-darkbrush``,
264 modules matched via the ReplicatedLinear projections) passed per-request through
``OmniDiffusionSamplingParams.lora_request`` / ``lora_scale`` — the same fields the
diffusion LoRA manager reads to activate/deactivate the adapter.
"""
lora_request = LoRARequest(lora_name="darkbrush", lora_int_id=stable_lora_int_id(LORA), lora_path=LORA)
baseline = _generate(omni_runner_handler, None)
img_1x = _generate(omni_runner_handler, lora_request, lora_scale=1.0)
img_2x = _generate(omni_runner_handler, lora_request, lora_scale=2.0)
restored = _generate(omni_runner_handler, None)
diff_1x = np.abs(baseline - img_1x).mean()
diff_2x = np.abs(baseline - img_2x).mean()
diff_restored = np.abs(baseline - restored).mean()
# (a) Adapter has a visible effect at both scales.
assert diff_1x > 0.5, f"LoRA scale=1.0 had no visible effect: diff={diff_1x}"
assert diff_2x > 0.5, f"LoRA scale=2.0 had no visible effect: diff={diff_2x}"
# (b) Scale changes the output (stronger adapter -> larger perturbation).
assert not np.isclose(diff_1x, diff_2x, atol=1.0), f"LoRA scale had no effect: 1x={diff_1x:.2f}, 2x={diff_2x:.2f}"
# (c) Passing lora_request=None cleanly deactivates: same seed -> back to baseline (modulo fp drift).
assert diff_restored < 5.0, f"LoRA did not deactivate cleanly: diff_restored={diff_restored:.2f}"
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""L4 online-serving tests for the Krea 2 text-to-image diffusion pipeline.
Exercises the OpenAI-compatible ``/v1/images/generations`` route (served via
``vllm serve <model> --omni --enforce-eager``) rather than the offline engine path
covered by ``tests/e2e/offline_inference/test_krea2_expansion.py``.
Runs against the public few-step distilled checkpoint ``krea/Krea-2-Turbo`` by default
(override with the ``KREA2_MODEL`` environment variable, e.g. ``krea/Krea-2-Raw`` for the
Raw checkpoint or a local diffusers directory). Krea 2 shares the Qwen-Image VAE and a
similar single-stream MMDiT, so this test mirrors the structure/fixtures/markers of
``test_qwen_image_expansion.py``.
Coverage mirrors the offline suite: a basic functional smoke plus the layerwise-CPU-offload
path that the pipeline declares via ``SupportsComponentDiscovery`` /
``_layerwise_offload_blocks_attrs``. Sizing is kept light (512x512, few steps) so the case
fits single-GPU H100 CI rather than the full 2048x2048 distilled resolution.
"""
import base64
import os
from io import BytesIO
import numpy as np
import pytest
import requests
from PIL import Image
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
MODEL = os.environ.get("KREA2_MODEL", "krea/Krea-2-Turbo")
# vLLM-Omni-compatible PEFT repackaging of krea/Krea-2-LoRA-darkbrush (264 modules, r=alpha=32).
LORA = os.environ.get("KREA2_LORA", "NagaSaiAbhinay/Krea-2-vllm-darkbrush-LoRA")
PROMPT = "a fox in the snow, photorealistic"
# darkbrush is a trigger-word LoRA; include its trigger phrase so the LoRA case has a strong signal.
LORA_PROMPT = "a fox in the snow, monochrome ink wash style"
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
def _get_diffusion_feature_cases(model: str):
"""Return single-card L4 online-serving cases for Krea 2.
Mirrors the offline suite: a basic functional smoke plus layerwise CPU offload.
"""
return [
# Basic functional smoke (single-card).
pytest.param(
OmniServerParams(
model=model,
server_args=["--enforce-eager"],
),
id="single_card_basic",
marks=SINGLE_CARD_FEATURE_MARKS,
),
# Layerwise CPU offload (single-card): SupportsComponentDiscovery +
# _layerwise_offload_blocks_attrs.
pytest.param(
OmniServerParams(
model=model,
server_args=["--enforce-eager", "--enable-layerwise-offload"],
),
id="single_card_layerwise_offload",
marks=SINGLE_CARD_FEATURE_MARKS,
),
]
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases(MODEL),
indirect=True,
)
def test_krea2(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Serve Krea 2 and validate the OpenAI-compatible image-generation route.
guidance_scale resolution is checkpoint-aware inside the pipeline (distilled -> no-CFG,
Raw -> CFG), so guidance_scale=0.0 stays agnostic to which checkpoint KREA2_MODEL points at.
Validation is delegated to assert_diffusion_response, which checks output dimensions and
basic correctness.
"""
messages = dummy_messages_from_mix_data(content_text=PROMPT)
request_config = {
"model": omni_server.model,
"messages": messages,
"extra_body": {
"height": 512,
"width": 512,
"num_inference_steps": 8,
"guidance_scale": 0.0,
"seed": 42,
},
}
openai_client.send_diffusion_request(request_config)
def _basic_lora_payload(lora: dict | None = None) -> dict:
payload = {
"prompt": LORA_PROMPT,
"n": 1,
"size": "512x512",
"num_inference_steps": 8,
"guidance_scale": 0.0,
"seed": 42,
}
if lora is not None:
payload["lora"] = lora
return payload
def _post_image(server: OmniServer, payload: dict) -> np.ndarray:
url = f"http://{server.host}:{server.port}/v1/images/generations"
resp = requests.post(url, json=payload, headers={"Authorization": "Bearer EMPTY"}, timeout=900)
resp.raise_for_status()
b64 = resp.json()["data"][0]["b64_json"]
img = Image.open(BytesIO(base64.b64decode(b64)))
img.load()
return np.asarray(img.convert("RGB"), dtype=np.int16)
@pytest.mark.parametrize(
"omni_server",
[
pytest.param(
OmniServerParams(model=MODEL, server_args=["--enforce-eager"]),
id="single_card_lora",
marks=SINGLE_CARD_FEATURE_MARKS,
)
],
indirect=True,
)
def test_krea2_lora(omni_server: OmniServer):
"""Serve Krea 2 and validate per-request LoRA over ``/v1/images/generations``.
Passes ``LORA`` via the top-level ``lora`` object (``{name, path, scale}``) the diffusion
image route parses into a ``LoRARequest`` — no server-side LoRA flag needed. Confirms the
adapter has a visible effect and that omitting ``lora`` cleanly restores the base output.
"""
baseline = _post_image(omni_server, _basic_lora_payload())
with_lora = _post_image(omni_server, _basic_lora_payload({"name": "darkbrush", "path": LORA, "scale": 1.0}))
restored = _post_image(omni_server, _basic_lora_payload())
diff_lora = np.abs(baseline - with_lora).mean()
diff_restored = np.abs(baseline - restored).mean()
assert diff_lora > 0.5, f"LoRA had no visible effect over the serving path: diff={diff_lora}"
assert diff_restored < 5.0, f"LoRA did not deactivate cleanly: diff_restored={diff_restored:.2f}"
@@ -56,6 +56,7 @@ EXCLUDED_MODELS = [
"NextStep11Pipeline",
"FluxPipeline",
"FluxDMD2Pipeline",
"Krea2Pipeline",
"QwenImageDMD2Pipeline",
"OmniGen2Pipeline",
"HeliosPipeline",