Add Cosmos3 model support (#3454)

Signed-off-by: Maciej Bala <mbala@nvidia.com>
Signed-off-by: MaciejBalaNV <mbala@nvidia.com>
Signed-off-by: lishunyang12 <lishunyang12@163.com>
Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com>
Co-authored-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
authored and GitHub committed 2026-06-02 04:04:31 +08:00
1 parent 2b7249fbb3
commit bc794e625f
29 files changed
+4095 -21

No files matched your search

@@ -51,6 +51,7 @@ class CuDNNAttentionImpl(AttentionImpl):
) -> None:
self.causal = causal
self.softmax_scale = softmax_scale
self.requires_gqa = num_heads != num_kv_heads
def forward_cuda(
self,
@@ -84,6 +85,7 @@ class CuDNNAttentionImpl(AttentionImpl):
dropout_p=0.0,
is_causal=self.causal,
scale=self.softmax_scale,
enable_gqa=self.requires_gqa,
)
except RuntimeError as e:
if "No available kernel" not in str(e):
@@ -100,5 +102,6 @@ class CuDNNAttentionImpl(AttentionImpl):
dropout_p=0.0,
is_causal=self.causal,
scale=self.softmax_scale,
enable_gqa=self.requires_gqa,
)
return output.permute(0, 2, 1, 3)
+79
View File
@@ -1725,6 +1725,84 @@ def enable_cache_for_hunyuan_video_15(pipeline: Any, cache_config: Any) -> Calla
return refresh_cache_context
def enable_cache_for_cosmos3(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
"""Enable cache-dit for Cosmos3.
Cosmos3 has a dual-pathway architecture (UND + GEN) but only the GEN
pathway (``gen_layers``) runs at every denoising step. The UND pathway
computes once and its K/V are cached by the pipeline itself; no cache-dit
needed there. We wrap only ``gen_layers`` via ``BlockAdapter``.
Args:
pipeline: The Cosmos3 pipeline instance.
cache_config: DiffusionCacheConfig instance with cache configuration.
Returns:
A refresh function that can be called to update cache context with new num_inference_steps.
"""
db_cache_config = _build_db_cache_config(cache_config)
calibrator_config = None
if cache_config.enable_taylorseer:
taylorseer_order = cache_config.taylorseer_order
calibrator_config = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order)
logger.info(f"TaylorSeer enabled with order={taylorseer_order}")
logger.info(
f"Enabling cache-dit on Cosmos3 gen_layers: "
f"Fn={db_cache_config.Fn_compute_blocks}, "
f"Bn={db_cache_config.Bn_compute_blocks}, "
f"W={db_cache_config.max_warmup_steps}, "
)
cache_dit.enable_cache(
BlockAdapter(
transformer=pipeline.transformer,
blocks=[pipeline.transformer.gen_layers],
# Cosmos3 GEN blocks return only hidden_states. Per-layer UND K/V
# conditioning uses the transformer's cache-dit fallback path.
forward_pattern=[ForwardPattern.Pattern_3],
params_modifiers=[
ParamsModifier(
cache_config=db_cache_config,
calibrator_config=calibrator_config,
),
],
check_forward_pattern=False,
has_separate_cfg=True,
),
cache_config=db_cache_config,
calibrator_config=calibrator_config,
)
# The T2I denoising loop skips the unconditional forward outside the
# guidance interval as a speed optimization. cache-dit distinguishes the
# conditional vs unconditional passes purely by transformer-forward parity
# (has_separate_cfg=True above), so that skip would desync its per-generation
# step accounting. Still do both cond/uncond CFG steps when cache-dit is active.
# CFG is instead neutralized via scale=1.0 outside the interval.
pipeline._cache_dit_requires_paired_cfg = True
def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None:
if cache_config.scm_steps_mask_policy is None:
cache_dit.refresh_context(pipeline.transformer, num_inference_steps=num_inference_steps, verbose=verbose)
else:
cache_dit.refresh_context(
pipeline.transformer,
cache_config=DBCacheConfig().reset(
num_inference_steps=num_inference_steps,
steps_computation_mask=cache_dit.steps_mask(
mask_policy=cache_config.scm_steps_mask_policy,
total_steps=num_inference_steps,
),
steps_computation_policy=cache_config.scm_steps_policy,
),
verbose=verbose,
)
return refresh_cache_context
# Register custom cache-dit enablers after function definitions
CUSTOM_DIT_ENABLERS.update(
{
@@ -1752,6 +1830,7 @@ CUSTOM_DIT_ENABLERS.update(
"HunyuanVideo15Pipeline": enable_cache_for_hunyuan_video_15,
"HunyuanVideo15I2VPipeline": enable_cache_for_hunyuan_video_15,
"HeliosPipeline": enable_cache_for_helios,
"Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3,
}
)
+14 -6
View File
@@ -83,6 +83,18 @@ def supports_audio_output(model_class_name: str) -> bool:
return bool(getattr(model_cls, "support_audio_output", False))
def _move_tensor_tree_to_cpu(value: object) -> object:
if isinstance(value, torch.Tensor):
return value.cpu() if value.device.type != "cpu" else value
if isinstance(value, dict):
return {key: _move_tensor_tree_to_cpu(item) for key, item in value.items()}
if isinstance(value, list):
return [_move_tensor_tree_to_cpu(item) for item in value]
if isinstance(value, tuple):
return tuple(_move_tensor_tree_to_cpu(item) for item in value)
return value
def get_dummy_run_num_frames(model_class_name: str, supports_audio_input: bool) -> int:
model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name)
if model_cls is not None and hasattr(model_cls, "dummy_run_num_frames"):
@@ -237,12 +249,8 @@ class DiffusionEngine:
# post-processing to avoid device OOM — model weights may still
# reside on the device and leave no headroom for intermediates.
output_data = output.output
if (
self.od_config.enable_cpu_offload
and isinstance(output_data, torch.Tensor)
and output_data.device.type != "cpu"
):
output_data = output_data.cpu()
if self.od_config.enable_cpu_offload:
output_data = _move_tensor_tree_to_cpu(output_data)
postprocess_start_time = time.perf_counter()
if self.post_process_func is not None:
+29 -3
View File
@@ -85,16 +85,42 @@ def _pack_tensor_if_large(val: torch.Tensor) -> torch.Tensor | dict:
return val
def _pack_value_if_large(val: object) -> object:
"""Recursively replace large tensors with SHM handles.
Walks the container shapes pipelines return as ``DiffusionOutput.output``:
bare tensors, dicts (e.g. Cosmos3 ``{"image"/"video": ...}``), and
tuples/lists (e.g. LTX2 and DreamID ``(video, audio)``). Other values pass
through unchanged. ``_unpack_if_shm_handle`` must mirror these shapes — keep
the two in sync.
"""
if isinstance(val, torch.Tensor):
return _pack_tensor_if_large(val)
if isinstance(val, dict):
return {key: _pack_value_if_large(value) for key, value in val.items()}
if isinstance(val, list):
return [_pack_value_if_large(item) for item in val]
if isinstance(val, tuple):
return tuple(_pack_value_if_large(item) for item in val)
return val
def _unpack_if_shm_handle(val: object) -> object:
"""Reconstruct a tensor from an SHM handle dict, or return as-is."""
"""Reconstruct tensors from SHM handles, mirroring ``_pack_value_if_large``."""
if isinstance(val, dict) and val.get("__tensor_shm__"):
return _tensor_from_shm(val)
if isinstance(val, dict):
return {key: _unpack_if_shm_handle(value) for key, value in val.items()}
if isinstance(val, list):
return [_unpack_if_shm_handle(item) for item in val]
if isinstance(val, tuple):
return tuple(_unpack_if_shm_handle(item) for item in val)
return val
def _pack_diffusion_fields(output: DiffusionOutput) -> DiffusionOutput:
if output.output is not None and isinstance(output.output, torch.Tensor):
output.output = _pack_tensor_if_large(output.output)
if output.output is not None:
output.output = _pack_value_if_large(output.output)
if output.trajectory_latents is not None and isinstance(output.trajectory_latents, torch.Tensor):
output.trajectory_latents = _pack_tensor_if_large(output.trajectory_latents)
if output.trajectory_timesteps is not None and isinstance(output.trajectory_timesteps, torch.Tensor):
@@ -0,0 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .pipeline_cosmos3 import (
Cosmos3OmniDiffusersPipeline,
get_cosmos3_post_process_func,
get_cosmos3_pre_process_func,
)
from .transformer_cosmos3 import Cosmos3VFMTransformer
__all__ = [
"Cosmos3OmniDiffusersPipeline",
"get_cosmos3_post_process_func",
"get_cosmos3_pre_process_func",
"Cosmos3VFMTransformer",
]
@@ -0,0 +1,185 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Cosmos3 guardrail hooks for vllm-omni.
Thin adapter around the ``cosmos_guardrail`` package's ``CosmosSafetyChecker``
(Blocklist + Qwen3Guard for text, RetinaFace face-blur for video).
Enabled by default. Disable server-wide with ``--cosmos3-no-guardrails`` (which
sets ``od_config.model_config["guardrails"] = False``); per-request overrides
ride on ``sampling_params.extra_args["guardrails"]``.
"""
from __future__ import annotations
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
import numpy as np
import torch
from vllm.logger import init_logger
from vllm_omni.diffusion.models.progress_bar import _is_rank_zero
if TYPE_CHECKING:
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
logger = init_logger(__name__)
try:
from cosmos_guardrail import CosmosSafetyChecker
_COSMOS_GUARDRAIL_AVAILABLE = True
except ImportError:
_COSMOS_GUARDRAIL_AVAILABLE = False
class CosmosSafetyChecker: # type: ignore[no-redef]
# Raised at runtime (not import time) so guardrail-less inference
# continues to work when ``cosmos_guardrail`` is not installed and
# ``model_config["guardrails"]`` is False.
def __init__(self, *args, **kwargs):
raise ValueError(
f"You have disabled the safety checker for {self.__class__}. This is in violation of the "
"[NVIDIA Open Model License Agreement]"
"(https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license). "
f"Please ensure that you are compliant with the license agreement."
f"Please install cosmos-guardrail package to enable safety checks."
)
TextGuardrailFn = Callable[[str], None]
VideoGuardrailFn = Callable[[np.ndarray], np.ndarray]
_text_guardrail: TextGuardrailFn | None = None
_video_guardrail: VideoGuardrailFn | None = None
# ---------------------------------------------------------------------------
# Default guardrail builders
# ---------------------------------------------------------------------------
def _nn_models(runner: Any) -> list[torch.nn.Module]:
return [m for m in runner.models if isinstance(m, torch.nn.Module)]
def _build_text_guardrail(checker: Any) -> TextGuardrailFn:
def text_guardrail(prompt: str) -> None:
if not checker.check_text_safety(prompt):
# CosmosSafetyChecker logs the specific reason at CRITICAL.
raise ValueError("Guardrail blocked prompt")
return text_guardrail
def _build_video_guardrail(checker: Any, offload_to_cpu: bool) -> VideoGuardrailFn:
video_models = _nn_models(checker.video_guardrail)
compute_device = "cuda"
def video_guardrail(frames: np.ndarray) -> np.ndarray:
if offload_to_cpu:
for m in video_models:
m.to(compute_device)
try:
result = checker.check_video_safety(frames)
finally:
if offload_to_cpu:
for m in video_models:
m.to("cpu")
# ``check_video_safety`` returns ``None`` when the content safety
# filter blocks the frames. The face-blur postprocessor (the only
# video module enabled by default) does not block, so in practice
# ``result`` is always an ndarray here.
return result if result is not None else frames
return video_guardrail
# ---------------------------------------------------------------------------
# Initialization
# ---------------------------------------------------------------------------
def _init_default_guardrails(offload_to_cpu: bool = False) -> None:
global _text_guardrail, _video_guardrail
if _text_guardrail is not None:
return
if _is_rank_zero():
logger.info("Initializing Cosmos3 guardrails (offload_to_cpu=%s)...", offload_to_cpu)
# Instantiation raises ValueError when ``cosmos_guardrail`` is not
# installed - this is the right moment to fail loudly because the
# caller has opted in to guardrails.
checker = CosmosSafetyChecker()
# Place text models on their resting device permanently. Video models
# idle on CPU when offload is on and move to GPU per-call (handled in
# the video guardrail closure).
idle_device = "cpu" if offload_to_cpu else "cuda"
for m in _nn_models(checker.text_guardrail):
m.to(idle_device)
for m in _nn_models(checker.video_guardrail):
m.to(idle_device)
_text_guardrail = _build_text_guardrail(checker)
_video_guardrail = _build_video_guardrail(checker, offload_to_cpu)
if _is_rank_zero():
logger.info("Cosmos3 guardrails initialized.")
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
def ensure_initialized(od_config: OmniDiffusionConfig) -> None:
if not is_guardrails_enabled(od_config):
return
model_config = od_config.model_config or {}
_init_default_guardrails(offload_to_cpu=bool(model_config.get("offload_guardrail_models", False)))
def check_text_safety(prompt: str) -> None:
if _text_guardrail is not None:
_text_guardrail(prompt)
def check_video_safety(video_tensor: torch.Tensor) -> torch.Tensor:
if _video_guardrail is None:
return video_tensor
v = video_tensor.detach().cpu().float()
if v.dim() == 5:
v = v[0]
v = v.clamp(-1, 1) * 0.5 + 0.5
frames_np = (v.permute(1, 2, 3, 0).numpy() * 255).round().astype(np.uint8)
frames_np = _video_guardrail(frames_np)
# Convert back to [-1, 1] to match the VAE output range.
result = torch.from_numpy(frames_np.copy()).float() / 127.5 - 1.0
result = result.permute(3, 0, 1, 2)
if video_tensor.dim() == 5:
result = result.unsqueeze(0)
return result.to(video_tensor.device)
def is_guardrails_enabled(
od_config: OmniDiffusionConfig,
sampling_params: OmniDiffusionSamplingParams | None = None,
) -> bool:
"""Resolve the active guardrail gate.
Server-level ``od_config.model_config["guardrails"]`` decides whether the
guardrail models are loaded at all (eager load at pipeline build time).
When that is False, no per-request override can turn checks back on,
because the singletons in this module are never populated.
When the server gate is on, ``sampling_params.extra_args["guardrails"]``
may override on a per-request basis: ``False`` skips the check for that
request, anything else (or missing) keeps the default behavior.
"""
model_config = od_config.model_config or {}
if not bool(model_config.get("guardrails", True)):
return False
if sampling_params is not None:
per_request = (sampling_params.extra_args or {}).get("guardrails")
if per_request is not None:
return bool(per_request)
return True
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+7
View File
@@ -266,6 +266,11 @@ _DIFFUSION_MODELS = {
"pipeline_omnivoice",
"OmniVoicePipeline",
),
"Cosmos3OmniDiffusersPipeline": (
"cosmos3",
"pipeline_cosmos3",
"Cosmos3OmniDiffusersPipeline",
),
"DiffusersAdapterPipeline": (
"diffusers_adapter",
"pipeline_diffusers_adapter",
@@ -498,6 +503,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = {
"OmniVoicePipeline": "get_omnivoice_post_process_func",
"DreamIDOmniPipeline": "get_dreamid_omni_post_process_func",
"SenseNovaU1Pipeline": "get_sensenova_u1_post_process_func",
"Cosmos3OmniDiffusersPipeline": "get_cosmos3_post_process_func",
"HiDreamImagePipeline": "get_hidream_image_post_process_func",
}
@@ -522,6 +528,7 @@ _DIFFUSION_PRE_PROCESS_FUNCS = {
"HunyuanVideo15ImageToVideoPipeline": "get_hunyuan_video_15_i2v_pre_process_func",
"HunyuanImage3ForCausalMM": "get_hunyuan_image_3_pre_process_func",
"MagiHumanPipeline": "get_magi_human_pre_process_func",
"Cosmos3OmniDiffusersPipeline": "get_cosmos3_pre_process_func",
}