[Model] Add Krea 2 text-to-image diffusion model (#4730)
Signed-off-by: Abhinay1997 <devarintinagasaiabhinay@gmail.com>
This commit is contained in:
1 parent
b95905deb1
commit
6581e2afa9
14 files changed
+1666
No files matched your search
@@ -0,0 +1,155 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""L4 functionality tests for the Krea 2 text-to-image diffusion pipeline (``Krea2Pipeline``).
|
||||
|
||||
Runs against the public few-step distilled checkpoint ``krea/Krea-2-Turbo`` by default (override with the
|
||||
``KREA2_MODEL`` environment variable, e.g. ``krea/Krea-2-Raw`` for the Raw checkpoint or a local diffusers
|
||||
directory). They cover a basic functional smoke plus the layerwise-CPU-offload path that the pipeline declares
|
||||
via ``SupportsComponentDiscovery`` / ``_layerwise_offload_blocks_attrs``.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunnerHandler
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.lora.utils import stable_lora_int_id
|
||||
|
||||
MODEL = os.environ.get("KREA2_MODEL", "krea/Krea-2-Turbo")
|
||||
# vLLM-Omni-compatible PEFT repackaging of krea/Krea-2-LoRA-darkbrush (264 modules, r=alpha=32).
|
||||
LORA = os.environ.get("KREA2_LORA", "NagaSaiAbhinay/Krea-2-vllm-darkbrush-LoRA")
|
||||
PROMPT = "a fox in the snow, photorealistic"
|
||||
# darkbrush is a trigger-word LoRA: the trained monochrome ink-wash style only manifests when
|
||||
# its trigger phrase is present, so the LoRA test prompt appends it for a strong, stable signal.
|
||||
LORA_TRIGGER = "monochrome ink wash style"
|
||||
LORA_PROMPT = f"a fox in the snow, {LORA_TRIGGER}"
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.full_model,
|
||||
]
|
||||
|
||||
|
||||
def _sampling() -> OmniDiffusionSamplingParams:
|
||||
# Small resolution + few steps to keep the L4 case light. guidance_scale resolution is checkpoint-aware inside
|
||||
# the pipeline (distilled -> no-CFG, Raw -> CFG), so this stays agnostic to which checkpoint KREA2_MODEL points at.
|
||||
return OmniDiffusionSamplingParams(
|
||||
height=512,
|
||||
width=512,
|
||||
num_inference_steps=8,
|
||||
guidance_scale=0.0,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
@pytest.mark.parametrize("omni_runner", [(MODEL, None)], indirect=True)
|
||||
def test_krea2_text_to_image_001(omni_runner_handler: OmniRunnerHandler) -> None:
|
||||
"""Basic functional smoke: a single prompt produces a decoded image."""
|
||||
omni_runner_handler.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
@pytest.mark.parametrize(
|
||||
"omni_runner",
|
||||
[(MODEL, None, {"enable_layerwise_offload": True})],
|
||||
indirect=True,
|
||||
)
|
||||
def test_krea2_layerwise_offload(omni_runner_handler: OmniRunnerHandler) -> None:
|
||||
"""Exercise layerwise CPU offload on the DiT (SupportsComponentDiscovery + _layerwise_offload_blocks_attrs)."""
|
||||
omni_runner_handler.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
@pytest.mark.cache
|
||||
@pytest.mark.parametrize("omni_runner", [(MODEL, None, {"cache_backend": "cache_dit"})], indirect=True)
|
||||
def test_krea2_cache_dit(omni_runner_handler: OmniRunnerHandler) -> None:
|
||||
"""Exercise Cache-DiT on Krea 2 via the custom Krea2Pipeline enabler (``enable_cache_for_krea2``).
|
||||
|
||||
Validates the docs' Cache-DiT support claim for Krea 2. ``has_separate_cfg`` is checkpoint-aware
|
||||
(False for the distilled Turbo no-CFG path, True for the Raw CFG path); the default Turbo checkpoint
|
||||
exercises the no-CFG branch.
|
||||
"""
|
||||
omni_runner_handler.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize(
|
||||
"omni_runner",
|
||||
# Standalone HSDP: use_hsdp=True with an explicit hsdp_shard_size shards the DiT weights
|
||||
# across 2 ranks (replicate_size defaults to 1 -> world_size = 1 * 2 = 2). HSDP must keep
|
||||
# tensor_parallel_size/data_parallel_size at 1 (see vllm_omni/diffusion/data.py __post_init__).
|
||||
# Also validated locally at 8 GPUs (hsdp_shard_size=8); CI shard provisions 2.
|
||||
[(MODEL, None, {"use_hsdp": True, "hsdp_shard_size": 2})],
|
||||
indirect=True,
|
||||
)
|
||||
def test_krea2_hsdp(omni_runner_handler: OmniRunnerHandler) -> None:
|
||||
"""Exercise Hybrid Sharded Data Parallel (HSDP) weight sharding on the DiT across 2 GPUs.
|
||||
|
||||
Validates the docs' HSDP-support claim for Krea 2. Scheduled on the multi-GPU (distributed_cuda)
|
||||
shard by ``num_cards=2``; skipped automatically on boxes with fewer than 2 CUDA devices.
|
||||
"""
|
||||
omni_runner_handler.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize(
|
||||
"omni_runner",
|
||||
# VAE patch-parallel decode: partition the VAE decode across the TP world. Mirrors the online
|
||||
# "tp_vae_patch" case (tensor_parallel_size=2 + vae_patch_parallel_size=2) from the wan2.2/z-image
|
||||
# expansion suites, so vae_patch_parallel_size must equal the world size established by TP.
|
||||
# Also validated locally at 8 GPUs (tp=8, vae_patch=8); CI shard provisions 2.
|
||||
[(MODEL, None, {"tensor_parallel_size": 2, "vae_patch_parallel_size": 2})],
|
||||
indirect=True,
|
||||
)
|
||||
def test_krea2_vae_patch_parallel(omni_runner_handler: OmniRunnerHandler) -> None:
|
||||
"""Exercise VAE patch-parallel (decode) across 2 GPUs.
|
||||
|
||||
Validates the docs' VAE-patch-parallel (decode) support claim for Krea 2. Scheduled on the
|
||||
multi-GPU (distributed_cuda) shard by ``num_cards=2``; skipped automatically on boxes with
|
||||
fewer than 2 CUDA devices.
|
||||
"""
|
||||
omni_runner_handler.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
|
||||
|
||||
|
||||
def _generate(handler: OmniRunnerHandler, lora_request: LoRARequest | None, lora_scale: float = 1.0) -> np.ndarray:
|
||||
sp = _sampling()
|
||||
sp.lora_request = lora_request
|
||||
sp.lora_scale = lora_scale
|
||||
resp = handler.send_diffusion_request({"model": MODEL, "prompt": LORA_PROMPT, "sampling_params": sp})
|
||||
return np.asarray(resp.images[0].convert("RGB"), dtype=np.int16)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
@pytest.mark.parametrize("omni_runner", [(MODEL, None)], indirect=True)
|
||||
def test_krea2_lora(omni_runner_handler: OmniRunnerHandler) -> None:
|
||||
"""Validate diffusion LoRA on Krea 2: visible effect, scale sensitivity, clean deactivation.
|
||||
|
||||
Uses ``LORA`` (a vLLM-Omni-compatible PEFT repackaging of ``krea/Krea-2-LoRA-darkbrush``,
|
||||
264 modules matched via the ReplicatedLinear projections) passed per-request through
|
||||
``OmniDiffusionSamplingParams.lora_request`` / ``lora_scale`` — the same fields the
|
||||
diffusion LoRA manager reads to activate/deactivate the adapter.
|
||||
"""
|
||||
lora_request = LoRARequest(lora_name="darkbrush", lora_int_id=stable_lora_int_id(LORA), lora_path=LORA)
|
||||
|
||||
baseline = _generate(omni_runner_handler, None)
|
||||
img_1x = _generate(omni_runner_handler, lora_request, lora_scale=1.0)
|
||||
img_2x = _generate(omni_runner_handler, lora_request, lora_scale=2.0)
|
||||
restored = _generate(omni_runner_handler, None)
|
||||
|
||||
diff_1x = np.abs(baseline - img_1x).mean()
|
||||
diff_2x = np.abs(baseline - img_2x).mean()
|
||||
diff_restored = np.abs(baseline - restored).mean()
|
||||
|
||||
# (a) Adapter has a visible effect at both scales.
|
||||
assert diff_1x > 0.5, f"LoRA scale=1.0 had no visible effect: diff={diff_1x}"
|
||||
assert diff_2x > 0.5, f"LoRA scale=2.0 had no visible effect: diff={diff_2x}"
|
||||
# (b) Scale changes the output (stronger adapter -> larger perturbation).
|
||||
assert not np.isclose(diff_1x, diff_2x, atol=1.0), f"LoRA scale had no effect: 1x={diff_1x:.2f}, 2x={diff_2x:.2f}"
|
||||
# (c) Passing lora_request=None cleanly deactivates: same seed -> back to baseline (modulo fp drift).
|
||||
assert diff_restored < 5.0, f"LoRA did not deactivate cleanly: diff_restored={diff_restored:.2f}"
|
||||
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""L4 online-serving tests for the Krea 2 text-to-image diffusion pipeline.
|
||||
|
||||
Exercises the OpenAI-compatible ``/v1/images/generations`` route (served via
|
||||
``vllm serve <model> --omni --enforce-eager``) rather than the offline engine path
|
||||
covered by ``tests/e2e/offline_inference/test_krea2_expansion.py``.
|
||||
|
||||
Runs against the public few-step distilled checkpoint ``krea/Krea-2-Turbo`` by default
|
||||
(override with the ``KREA2_MODEL`` environment variable, e.g. ``krea/Krea-2-Raw`` for the
|
||||
Raw checkpoint or a local diffusers directory). Krea 2 shares the Qwen-Image VAE and a
|
||||
similar single-stream MMDiT, so this test mirrors the structure/fixtures/markers of
|
||||
``test_qwen_image_expansion.py``.
|
||||
|
||||
Coverage mirrors the offline suite: a basic functional smoke plus the layerwise-CPU-offload
|
||||
path that the pipeline declares via ``SupportsComponentDiscovery`` /
|
||||
``_layerwise_offload_blocks_attrs``. Sizing is kept light (512x512, few steps) so the case
|
||||
fits single-GPU H100 CI rather than the full 2048x2048 distilled resolution.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import os
|
||||
from io import BytesIO
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
|
||||
MODEL = os.environ.get("KREA2_MODEL", "krea/Krea-2-Turbo")
|
||||
# vLLM-Omni-compatible PEFT repackaging of krea/Krea-2-LoRA-darkbrush (264 modules, r=alpha=32).
|
||||
LORA = os.environ.get("KREA2_LORA", "NagaSaiAbhinay/Krea-2-vllm-darkbrush-LoRA")
|
||||
PROMPT = "a fox in the snow, photorealistic"
|
||||
# darkbrush is a trigger-word LoRA; include its trigger phrase so the LoRA case has a strong signal.
|
||||
LORA_PROMPT = "a fox in the snow, monochrome ink wash style"
|
||||
|
||||
SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
|
||||
|
||||
|
||||
def _get_diffusion_feature_cases(model: str):
|
||||
"""Return single-card L4 online-serving cases for Krea 2.
|
||||
|
||||
Mirrors the offline suite: a basic functional smoke plus layerwise CPU offload.
|
||||
"""
|
||||
return [
|
||||
# Basic functional smoke (single-card).
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=["--enforce-eager"],
|
||||
),
|
||||
id="single_card_basic",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
),
|
||||
# Layerwise CPU offload (single-card): SupportsComponentDiscovery +
|
||||
# _layerwise_offload_blocks_attrs.
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=model,
|
||||
server_args=["--enforce-eager", "--enable-layerwise-offload"],
|
||||
),
|
||||
id="single_card_layerwise_offload",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases(MODEL),
|
||||
indirect=True,
|
||||
)
|
||||
def test_krea2(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Serve Krea 2 and validate the OpenAI-compatible image-generation route.
|
||||
|
||||
guidance_scale resolution is checkpoint-aware inside the pipeline (distilled -> no-CFG,
|
||||
Raw -> CFG), so guidance_scale=0.0 stays agnostic to which checkpoint KREA2_MODEL points at.
|
||||
Validation is delegated to assert_diffusion_response, which checks output dimensions and
|
||||
basic correctness.
|
||||
"""
|
||||
messages = dummy_messages_from_mix_data(content_text=PROMPT)
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"extra_body": {
|
||||
"height": 512,
|
||||
"width": 512,
|
||||
"num_inference_steps": 8,
|
||||
"guidance_scale": 0.0,
|
||||
"seed": 42,
|
||||
},
|
||||
}
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
def _basic_lora_payload(lora: dict | None = None) -> dict:
|
||||
payload = {
|
||||
"prompt": LORA_PROMPT,
|
||||
"n": 1,
|
||||
"size": "512x512",
|
||||
"num_inference_steps": 8,
|
||||
"guidance_scale": 0.0,
|
||||
"seed": 42,
|
||||
}
|
||||
if lora is not None:
|
||||
payload["lora"] = lora
|
||||
return payload
|
||||
|
||||
|
||||
def _post_image(server: OmniServer, payload: dict) -> np.ndarray:
|
||||
url = f"http://{server.host}:{server.port}/v1/images/generations"
|
||||
resp = requests.post(url, json=payload, headers={"Authorization": "Bearer EMPTY"}, timeout=900)
|
||||
resp.raise_for_status()
|
||||
b64 = resp.json()["data"][0]["b64_json"]
|
||||
img = Image.open(BytesIO(base64.b64decode(b64)))
|
||||
img.load()
|
||||
return np.asarray(img.convert("RGB"), dtype=np.int16)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
[
|
||||
pytest.param(
|
||||
OmniServerParams(model=MODEL, server_args=["--enforce-eager"]),
|
||||
id="single_card_lora",
|
||||
marks=SINGLE_CARD_FEATURE_MARKS,
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
def test_krea2_lora(omni_server: OmniServer):
|
||||
"""Serve Krea 2 and validate per-request LoRA over ``/v1/images/generations``.
|
||||
|
||||
Passes ``LORA`` via the top-level ``lora`` object (``{name, path, scale}``) the diffusion
|
||||
image route parses into a ``LoRARequest`` — no server-side LoRA flag needed. Confirms the
|
||||
adapter has a visible effect and that omitting ``lora`` cleanly restores the base output.
|
||||
"""
|
||||
baseline = _post_image(omni_server, _basic_lora_payload())
|
||||
with_lora = _post_image(omni_server, _basic_lora_payload({"name": "darkbrush", "path": LORA, "scale": 1.0}))
|
||||
restored = _post_image(omni_server, _basic_lora_payload())
|
||||
|
||||
diff_lora = np.abs(baseline - with_lora).mean()
|
||||
diff_restored = np.abs(baseline - restored).mean()
|
||||
|
||||
assert diff_lora > 0.5, f"LoRA had no visible effect over the serving path: diff={diff_lora}"
|
||||
assert diff_restored < 5.0, f"LoRA did not deactivate cleanly: diff_restored={diff_restored:.2f}"
|
||||
@@ -56,6 +56,7 @@ EXCLUDED_MODELS = [
|
||||
"NextStep11Pipeline",
|
||||
"FluxPipeline",
|
||||
"FluxDMD2Pipeline",
|
||||
"Krea2Pipeline",
|
||||
"QwenImageDMD2Pipeline",
|
||||
"OmniGen2Pipeline",
|
||||
"HeliosPipeline",
|
||||
|
||||
Reference in new issue
Block a user