Add Cosmos3 model support (#3454)
Signed-off-by: Maciej Bala <mbala@nvidia.com> Signed-off-by: MaciejBalaNV <mbala@nvidia.com> Signed-off-by: lishunyang12 <lishunyang12@163.com> Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com> Co-authored-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
29 files changed
+4095
-21
No files matched your search
@@ -51,6 +51,7 @@ class CuDNNAttentionImpl(AttentionImpl):
|
||||
) -> None:
|
||||
self.causal = causal
|
||||
self.softmax_scale = softmax_scale
|
||||
self.requires_gqa = num_heads != num_kv_heads
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
@@ -84,6 +85,7 @@ class CuDNNAttentionImpl(AttentionImpl):
|
||||
dropout_p=0.0,
|
||||
is_causal=self.causal,
|
||||
scale=self.softmax_scale,
|
||||
enable_gqa=self.requires_gqa,
|
||||
)
|
||||
except RuntimeError as e:
|
||||
if "No available kernel" not in str(e):
|
||||
@@ -100,5 +102,6 @@ class CuDNNAttentionImpl(AttentionImpl):
|
||||
dropout_p=0.0,
|
||||
is_causal=self.causal,
|
||||
scale=self.softmax_scale,
|
||||
enable_gqa=self.requires_gqa,
|
||||
)
|
||||
return output.permute(0, 2, 1, 3)
|
||||
+79
@@ -1725,6 +1725,84 @@ def enable_cache_for_hunyuan_video_15(pipeline: Any, cache_config: Any) -> Calla
|
||||
return refresh_cache_context
|
||||
|
||||
|
||||
def enable_cache_for_cosmos3(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
|
||||
"""Enable cache-dit for Cosmos3.
|
||||
|
||||
Cosmos3 has a dual-pathway architecture (UND + GEN) but only the GEN
|
||||
pathway (``gen_layers``) runs at every denoising step. The UND pathway
|
||||
computes once and its K/V are cached by the pipeline itself; no cache-dit
|
||||
needed there. We wrap only ``gen_layers`` via ``BlockAdapter``.
|
||||
|
||||
Args:
|
||||
pipeline: The Cosmos3 pipeline instance.
|
||||
cache_config: DiffusionCacheConfig instance with cache configuration.
|
||||
|
||||
Returns:
|
||||
A refresh function that can be called to update cache context with new num_inference_steps.
|
||||
"""
|
||||
db_cache_config = _build_db_cache_config(cache_config)
|
||||
|
||||
calibrator_config = None
|
||||
if cache_config.enable_taylorseer:
|
||||
taylorseer_order = cache_config.taylorseer_order
|
||||
calibrator_config = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
|
||||
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
|
||||
|
||||
logger.info(
|
||||
f"Enabling cache-dit on Cosmos3 gen_layers: "
|
||||
f"Fn={db_cache_config.Fn_compute_blocks}, "
|
||||
f"Bn={db_cache_config.Bn_compute_blocks}, "
|
||||
f"W={db_cache_config.max_warmup_steps}, "
|
||||
)
|
||||
|
||||
cache_dit.enable_cache(
|
||||
BlockAdapter(
|
||||
transformer=pipeline.transformer,
|
||||
blocks=[pipeline.transformer.gen_layers],
|
||||
# Cosmos3 GEN blocks return only hidden_states. Per-layer UND K/V
|
||||
# conditioning uses the transformer's cache-dit fallback path.
|
||||
forward_pattern=[ForwardPattern.Pattern_3],
|
||||
params_modifiers=[
|
||||
ParamsModifier(
|
||||
cache_config=db_cache_config,
|
||||
calibrator_config=calibrator_config,
|
||||
),
|
||||
],
|
||||
check_forward_pattern=False,
|
||||
has_separate_cfg=True,
|
||||
),
|
||||
cache_config=db_cache_config,
|
||||
calibrator_config=calibrator_config,
|
||||
)
|
||||
|
||||
# The T2I denoising loop skips the unconditional forward outside the
|
||||
# guidance interval as a speed optimization. cache-dit distinguishes the
|
||||
# conditional vs unconditional passes purely by transformer-forward parity
|
||||
# (has_separate_cfg=True above), so that skip would desync its per-generation
|
||||
# step accounting. Still do both cond/uncond CFG steps when cache-dit is active.
|
||||
# CFG is instead neutralized via scale=1.0 outside the interval.
|
||||
pipeline._cache_dit_requires_paired_cfg = True
|
||||
|
||||
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
|
||||
if cache_config.scm_steps_mask_policy is None:
|
||||
cache_dit.refresh_context(pipeline.transformer, num_inference_steps=num_inference_steps, verbose=verbose)
|
||||
else:
|
||||
cache_dit.refresh_context(
|
||||
pipeline.transformer,
|
||||
cache_config=DBCacheConfig().reset(
|
||||
num_inference_steps=num_inference_steps,
|
||||
steps_computation_mask=cache_dit.steps_mask(
|
||||
mask_policy=cache_config.scm_steps_mask_policy,
|
||||
total_steps=num_inference_steps,
|
||||
),
|
||||
steps_computation_policy=cache_config.scm_steps_policy,
|
||||
),
|
||||
verbose=verbose,
|
||||
)
|
||||
|
||||
return refresh_cache_context
|
||||
|
||||
|
||||
# Register custom cache-dit enablers after function definitions
|
||||
CUSTOM_DIT_ENABLERS.update(
|
||||
{
|
||||
@@ -1752,6 +1830,7 @@ CUSTOM_DIT_ENABLERS.update(
|
||||
"HunyuanVideo15Pipeline": enable_cache_for_hunyuan_video_15,
|
||||
"HunyuanVideo15I2VPipeline": enable_cache_for_hunyuan_video_15,
|
||||
"HeliosPipeline": enable_cache_for_helios,
|
||||
"Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -83,6 +83,18 @@ def supports_audio_output(model_class_name: str) -> bool:
|
||||
return bool(getattr(model_cls, "support_audio_output", False))
|
||||
|
||||
|
||||
def _move_tensor_tree_to_cpu(value: object) -> object:
|
||||
if isinstance(value, torch.Tensor):
|
||||
return value.cpu() if value.device.type != "cpu" else value
|
||||
if isinstance(value, dict):
|
||||
return {key: _move_tensor_tree_to_cpu(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_move_tensor_tree_to_cpu(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(_move_tensor_tree_to_cpu(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def get_dummy_run_num_frames(model_class_name: str, supports_audio_input: bool) -> int:
|
||||
model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name)
|
||||
if model_cls is not None and hasattr(model_cls, "dummy_run_num_frames"):
|
||||
@@ -237,12 +249,8 @@ class DiffusionEngine:
|
||||
# post-processing to avoid device OOM — model weights may still
|
||||
# reside on the device and leave no headroom for intermediates.
|
||||
output_data = output.output
|
||||
if (
|
||||
self.od_config.enable_cpu_offload
|
||||
and isinstance(output_data, torch.Tensor)
|
||||
and output_data.device.type != "cpu"
|
||||
):
|
||||
output_data = output_data.cpu()
|
||||
if self.od_config.enable_cpu_offload:
|
||||
output_data = _move_tensor_tree_to_cpu(output_data)
|
||||
|
||||
postprocess_start_time = time.perf_counter()
|
||||
if self.post_process_func is not None:
|
||||
|
||||
@@ -85,16 +85,42 @@ def _pack_tensor_if_large(val: torch.Tensor) -> torch.Tensor | dict:
|
||||
return val
|
||||
|
||||
|
||||
def _pack_value_if_large(val: object) -> object:
|
||||
"""Recursively replace large tensors with SHM handles.
|
||||
|
||||
Walks the container shapes pipelines return as ``DiffusionOutput.output``:
|
||||
bare tensors, dicts (e.g. Cosmos3 ``{"image"/"video": ...}``), and
|
||||
tuples/lists (e.g. LTX2 and DreamID ``(video, audio)``). Other values pass
|
||||
through unchanged. ``_unpack_if_shm_handle`` must mirror these shapes — keep
|
||||
the two in sync.
|
||||
"""
|
||||
if isinstance(val, torch.Tensor):
|
||||
return _pack_tensor_if_large(val)
|
||||
if isinstance(val, dict):
|
||||
return {key: _pack_value_if_large(value) for key, value in val.items()}
|
||||
if isinstance(val, list):
|
||||
return [_pack_value_if_large(item) for item in val]
|
||||
if isinstance(val, tuple):
|
||||
return tuple(_pack_value_if_large(item) for item in val)
|
||||
return val
|
||||
|
||||
|
||||
def _unpack_if_shm_handle(val: object) -> object:
|
||||
"""Reconstruct a tensor from an SHM handle dict, or return as-is."""
|
||||
"""Reconstruct tensors from SHM handles, mirroring ``_pack_value_if_large``."""
|
||||
if isinstance(val, dict) and val.get("__tensor_shm__"):
|
||||
return _tensor_from_shm(val)
|
||||
if isinstance(val, dict):
|
||||
return {key: _unpack_if_shm_handle(value) for key, value in val.items()}
|
||||
if isinstance(val, list):
|
||||
return [_unpack_if_shm_handle(item) for item in val]
|
||||
if isinstance(val, tuple):
|
||||
return tuple(_unpack_if_shm_handle(item) for item in val)
|
||||
return val
|
||||
|
||||
|
||||
def _pack_diffusion_fields(output: DiffusionOutput) -> DiffusionOutput:
|
||||
if output.output is not None and isinstance(output.output, torch.Tensor):
|
||||
output.output = _pack_tensor_if_large(output.output)
|
||||
if output.output is not None:
|
||||
output.output = _pack_value_if_large(output.output)
|
||||
if output.trajectory_latents is not None and isinstance(output.trajectory_latents, torch.Tensor):
|
||||
output.trajectory_latents = _pack_tensor_if_large(output.trajectory_latents)
|
||||
if output.trajectory_timesteps is not None and isinstance(output.trajectory_timesteps, torch.Tensor):
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .pipeline_cosmos3 import (
|
||||
Cosmos3OmniDiffusersPipeline,
|
||||
get_cosmos3_post_process_func,
|
||||
get_cosmos3_pre_process_func,
|
||||
)
|
||||
from .transformer_cosmos3 import Cosmos3VFMTransformer
|
||||
|
||||
__all__ = [
|
||||
"Cosmos3OmniDiffusersPipeline",
|
||||
"get_cosmos3_post_process_func",
|
||||
"get_cosmos3_pre_process_func",
|
||||
"Cosmos3VFMTransformer",
|
||||
]
|
||||
@@ -0,0 +1,185 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cosmos3 guardrail hooks for vllm-omni.
|
||||
|
||||
Thin adapter around the ``cosmos_guardrail`` package's ``CosmosSafetyChecker``
|
||||
(Blocklist + Qwen3Guard for text, RetinaFace face-blur for video).
|
||||
|
||||
Enabled by default. Disable server-wide with ``--cosmos3-no-guardrails`` (which
|
||||
sets ``od_config.model_config["guardrails"] = False``); per-request overrides
|
||||
ride on ``sampling_params.extra_args["guardrails"]``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.models.progress_bar import _is_rank_zero
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
try:
|
||||
from cosmos_guardrail import CosmosSafetyChecker
|
||||
|
||||
_COSMOS_GUARDRAIL_AVAILABLE = True
|
||||
except ImportError:
|
||||
_COSMOS_GUARDRAIL_AVAILABLE = False
|
||||
|
||||
class CosmosSafetyChecker: # type: ignore[no-redef]
|
||||
# Raised at runtime (not import time) so guardrail-less inference
|
||||
# continues to work when ``cosmos_guardrail`` is not installed and
|
||||
# ``model_config["guardrails"]`` is False.
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise ValueError(
|
||||
f"You have disabled the safety checker for {self.__class__}. This is in violation of the "
|
||||
"[NVIDIA Open Model License Agreement]"
|
||||
"(https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license). "
|
||||
f"Please ensure that you are compliant with the license agreement."
|
||||
f"Please install cosmos-guardrail package to enable safety checks."
|
||||
)
|
||||
|
||||
|
||||
TextGuardrailFn = Callable[[str], None]
|
||||
VideoGuardrailFn = Callable[[np.ndarray], np.ndarray]
|
||||
|
||||
_text_guardrail: TextGuardrailFn | None = None
|
||||
_video_guardrail: VideoGuardrailFn | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default guardrail builders
|
||||
# ---------------------------------------------------------------------------
|
||||
def _nn_models(runner: Any) -> list[torch.nn.Module]:
|
||||
return [m for m in runner.models if isinstance(m, torch.nn.Module)]
|
||||
|
||||
|
||||
def _build_text_guardrail(checker: Any) -> TextGuardrailFn:
|
||||
def text_guardrail(prompt: str) -> None:
|
||||
if not checker.check_text_safety(prompt):
|
||||
# CosmosSafetyChecker logs the specific reason at CRITICAL.
|
||||
raise ValueError("Guardrail blocked prompt")
|
||||
|
||||
return text_guardrail
|
||||
|
||||
|
||||
def _build_video_guardrail(checker: Any, offload_to_cpu: bool) -> VideoGuardrailFn:
|
||||
video_models = _nn_models(checker.video_guardrail)
|
||||
compute_device = "cuda"
|
||||
|
||||
def video_guardrail(frames: np.ndarray) -> np.ndarray:
|
||||
if offload_to_cpu:
|
||||
for m in video_models:
|
||||
m.to(compute_device)
|
||||
try:
|
||||
result = checker.check_video_safety(frames)
|
||||
finally:
|
||||
if offload_to_cpu:
|
||||
for m in video_models:
|
||||
m.to("cpu")
|
||||
# ``check_video_safety`` returns ``None`` when the content safety
|
||||
# filter blocks the frames. The face-blur postprocessor (the only
|
||||
# video module enabled by default) does not block, so in practice
|
||||
# ``result`` is always an ndarray here.
|
||||
return result if result is not None else frames
|
||||
|
||||
return video_guardrail
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Initialization
|
||||
# ---------------------------------------------------------------------------
|
||||
def _init_default_guardrails(offload_to_cpu: bool = False) -> None:
|
||||
global _text_guardrail, _video_guardrail
|
||||
if _text_guardrail is not None:
|
||||
return
|
||||
if _is_rank_zero():
|
||||
logger.info("Initializing Cosmos3 guardrails (offload_to_cpu=%s)...", offload_to_cpu)
|
||||
|
||||
# Instantiation raises ValueError when ``cosmos_guardrail`` is not
|
||||
# installed - this is the right moment to fail loudly because the
|
||||
# caller has opted in to guardrails.
|
||||
checker = CosmosSafetyChecker()
|
||||
|
||||
# Place text models on their resting device permanently. Video models
|
||||
# idle on CPU when offload is on and move to GPU per-call (handled in
|
||||
# the video guardrail closure).
|
||||
idle_device = "cpu" if offload_to_cpu else "cuda"
|
||||
for m in _nn_models(checker.text_guardrail):
|
||||
m.to(idle_device)
|
||||
for m in _nn_models(checker.video_guardrail):
|
||||
m.to(idle_device)
|
||||
|
||||
_text_guardrail = _build_text_guardrail(checker)
|
||||
_video_guardrail = _build_video_guardrail(checker, offload_to_cpu)
|
||||
if _is_rank_zero():
|
||||
logger.info("Cosmos3 guardrails initialized.")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
def ensure_initialized(od_config: OmniDiffusionConfig) -> None:
|
||||
if not is_guardrails_enabled(od_config):
|
||||
return
|
||||
model_config = od_config.model_config or {}
|
||||
_init_default_guardrails(offload_to_cpu=bool(model_config.get("offload_guardrail_models", False)))
|
||||
|
||||
|
||||
def check_text_safety(prompt: str) -> None:
|
||||
if _text_guardrail is not None:
|
||||
_text_guardrail(prompt)
|
||||
|
||||
|
||||
def check_video_safety(video_tensor: torch.Tensor) -> torch.Tensor:
|
||||
if _video_guardrail is None:
|
||||
return video_tensor
|
||||
|
||||
v = video_tensor.detach().cpu().float()
|
||||
if v.dim() == 5:
|
||||
v = v[0]
|
||||
v = v.clamp(-1, 1) * 0.5 + 0.5
|
||||
frames_np = (v.permute(1, 2, 3, 0).numpy() * 255).round().astype(np.uint8)
|
||||
|
||||
frames_np = _video_guardrail(frames_np)
|
||||
|
||||
# Convert back to [-1, 1] to match the VAE output range.
|
||||
result = torch.from_numpy(frames_np.copy()).float() / 127.5 - 1.0
|
||||
result = result.permute(3, 0, 1, 2)
|
||||
if video_tensor.dim() == 5:
|
||||
result = result.unsqueeze(0)
|
||||
return result.to(video_tensor.device)
|
||||
|
||||
|
||||
def is_guardrails_enabled(
|
||||
od_config: OmniDiffusionConfig,
|
||||
sampling_params: OmniDiffusionSamplingParams | None = None,
|
||||
) -> bool:
|
||||
"""Resolve the active guardrail gate.
|
||||
|
||||
Server-level ``od_config.model_config["guardrails"]`` decides whether the
|
||||
guardrail models are loaded at all (eager load at pipeline build time).
|
||||
When that is False, no per-request override can turn checks back on,
|
||||
because the singletons in this module are never populated.
|
||||
|
||||
When the server gate is on, ``sampling_params.extra_args["guardrails"]``
|
||||
may override on a per-request basis: ``False`` skips the check for that
|
||||
request, anything else (or missing) keeps the default behavior.
|
||||
"""
|
||||
model_config = od_config.model_config or {}
|
||||
if not bool(model_config.get("guardrails", True)):
|
||||
return False
|
||||
if sampling_params is not None:
|
||||
per_request = (sampling_params.extra_args or {}).get("guardrails")
|
||||
if per_request is not None:
|
||||
return bool(per_request)
|
||||
return True
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -266,6 +266,11 @@ _DIFFUSION_MODELS = {
|
||||
"pipeline_omnivoice",
|
||||
"OmniVoicePipeline",
|
||||
),
|
||||
"Cosmos3OmniDiffusersPipeline": (
|
||||
"cosmos3",
|
||||
"pipeline_cosmos3",
|
||||
"Cosmos3OmniDiffusersPipeline",
|
||||
),
|
||||
"DiffusersAdapterPipeline": (
|
||||
"diffusers_adapter",
|
||||
"pipeline_diffusers_adapter",
|
||||
@@ -498,6 +503,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = {
|
||||
"OmniVoicePipeline": "get_omnivoice_post_process_func",
|
||||
"DreamIDOmniPipeline": "get_dreamid_omni_post_process_func",
|
||||
"SenseNovaU1Pipeline": "get_sensenova_u1_post_process_func",
|
||||
"Cosmos3OmniDiffusersPipeline": "get_cosmos3_post_process_func",
|
||||
"HiDreamImagePipeline": "get_hidream_image_post_process_func",
|
||||
}
|
||||
|
||||
@@ -522,6 +528,7 @@ _DIFFUSION_PRE_PROCESS_FUNCS = {
|
||||
"HunyuanVideo15ImageToVideoPipeline": "get_hunyuan_video_15_i2v_pre_process_func",
|
||||
"HunyuanImage3ForCausalMM": "get_hunyuan_image_3_pre_process_func",
|
||||
"MagiHumanPipeline": "get_magi_human_pre_process_func",
|
||||
"Cosmos3OmniDiffusersPipeline": "get_cosmos3_pre_process_func",
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user