[Model][Cosmos3] Keep diffusion sampling state in FP32 (#7592)
Signed-off-by: Yuliya Zhautouskaya <yzhautouskay@nvidia.com>
This commit is contained in:
1 parent
4af28f33bd
commit
27a6321a63
2 files changed
+97
-17
No files matched your search
@@ -78,6 +78,12 @@ def test_component_selective_model_offload_fails_before_component_loading(monkey
|
||||
pipeline_module.Cosmos3OmniDiffusersPipeline(od_config=config)
|
||||
|
||||
|
||||
def test_sampling_dtype_defaults_to_float32() -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
|
||||
|
||||
assert Cosmos3OmniDiffusersPipeline.sampling_dtype == torch.float32
|
||||
|
||||
|
||||
class StubScheduler:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -228,6 +234,7 @@ class StubCosmos3Transformer(nn.Module):
|
||||
{
|
||||
"token": token,
|
||||
"has_control": control_latents is not None,
|
||||
"hidden_states_dtype": hidden_states.dtype,
|
||||
"timestep": timestep.clone(),
|
||||
"text_mask": text_mask.clone(),
|
||||
"cache_before": self.cached_kv,
|
||||
@@ -2149,6 +2156,43 @@ def test_prepare_latents_for_video_image_sound_and_action(make_cosmos3_pipeline)
|
||||
torch.testing.assert_close(action, clean)
|
||||
|
||||
|
||||
def test_sampling_state_casts_transformer_execution_to_model_dtype(make_cosmos3_pipeline) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
pipeline.dtype = torch.bfloat16
|
||||
|
||||
latents = pipeline._prepare_latents(16, 24, 5, torch.Generator(device="cpu").manual_seed(0))
|
||||
assert pipeline.sampling_dtype == torch.float32
|
||||
assert latents.dtype == torch.float32
|
||||
|
||||
prediction = pipeline.predict_noise(
|
||||
hidden_states=latents,
|
||||
timestep=torch.tensor([1]),
|
||||
text_ids=_ids(2),
|
||||
text_mask=_mask(),
|
||||
)
|
||||
|
||||
assert pipeline.transformer.calls[-1]["hidden_states_dtype"] == torch.bfloat16
|
||||
assert prediction.dtype == torch.float32
|
||||
|
||||
|
||||
def test_sampling_dtype_can_use_model_dtype(make_cosmos3_pipeline) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
pipeline.dtype = torch.bfloat16
|
||||
pipeline.sampling_dtype = pipeline.dtype
|
||||
|
||||
latents = pipeline._prepare_latents(16, 24, 5, torch.Generator(device="cpu").manual_seed(0))
|
||||
prediction = pipeline.predict_noise(
|
||||
hidden_states=latents,
|
||||
timestep=torch.tensor([1]),
|
||||
text_ids=_ids(2),
|
||||
text_mask=_mask(),
|
||||
)
|
||||
|
||||
assert latents.dtype == torch.bfloat16
|
||||
assert pipeline.transformer.calls[-1]["hidden_states_dtype"] == torch.bfloat16
|
||||
assert prediction.dtype == torch.bfloat16
|
||||
|
||||
|
||||
def test_prepare_latents_i2v_encodes_only_conditioning_frame(make_cosmos3_pipeline) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
calls: list[tuple[str, tuple[int, ...]]] = []
|
||||
@@ -2232,6 +2276,7 @@ def test_prepare_inverse_dynamics_latents_encodes_full_video(make_cosmos3_pipeli
|
||||
|
||||
def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> None:
|
||||
pipeline = make_cosmos3_pipeline()
|
||||
pipeline.dtype = torch.bfloat16
|
||||
latents = torch.zeros(1, 2, 1, 1, 1)
|
||||
|
||||
result = pipeline.diffuse(
|
||||
@@ -2246,6 +2291,8 @@ def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> N
|
||||
guidance_interval=(500.0, 1000.0),
|
||||
)
|
||||
assert [call["token"] for call in pipeline.transformer.calls] == [2, 1, 2]
|
||||
assert all(call["hidden_states_dtype"] == torch.bfloat16 for call in pipeline.transformer.calls)
|
||||
assert result.dtype == torch.float32
|
||||
torch.testing.assert_close(result, torch.full_like(latents, 6.0))
|
||||
|
||||
i2v = pipeline.diffuse(
|
||||
@@ -2260,6 +2307,8 @@ def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> N
|
||||
velocity_mask=torch.tensor([[[[[0.0]], [[1.0]]]]]),
|
||||
image_latent=torch.full((1, 2, 1, 1, 1), 7.0),
|
||||
)
|
||||
assert pipeline.transformer.calls[-1]["hidden_states_dtype"] == torch.bfloat16
|
||||
assert i2v.dtype == torch.float32
|
||||
torch.testing.assert_close(i2v[:, :, 0:1], torch.full((1, 2, 1, 1, 1), 7.0))
|
||||
i2v_noise = pipeline.scheduler.step_calls[-1][0]
|
||||
torch.testing.assert_close(i2v_noise[:, :, 0:1], torch.zeros(1, 2, 1, 1, 1))
|
||||
@@ -2279,6 +2328,9 @@ def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> N
|
||||
guidance_scale=1.0,
|
||||
shared_kwargs={"video_shape": (1, 1, 1), "fps": 24.0, "action_domain_ids": torch.tensor([0])},
|
||||
)
|
||||
assert all(call["hidden_states_dtype"] == torch.bfloat16 for call in pipeline.transformer.calls)
|
||||
assert video_result.dtype == torch.float32
|
||||
assert action_result.dtype == torch.float32
|
||||
torch.testing.assert_close(video_result, torch.full_like(latents, 4.0))
|
||||
torch.testing.assert_close(action_result, torch.full((), 44.0).expand_as(action_result))
|
||||
|
||||
|
||||
@@ -932,6 +932,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
_encoder_modules: ClassVar[list[str]] = []
|
||||
_vae_modules: ClassVar[list[str]] = ["vae"]
|
||||
_resident_modules: ClassVar[list[str]] = []
|
||||
sampling_dtype: ClassVar[torch.dtype] = torch.float32
|
||||
|
||||
@classmethod
|
||||
def reference_video_decode_spec(
|
||||
@@ -1332,8 +1333,35 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
The transformer returns the raw prediction: video-only as a tensor,
|
||||
or a tuple in video, action, sound order for multimodal generation.
|
||||
"""
|
||||
|
||||
def _to_model_dtype(value: Any) -> Any:
|
||||
if isinstance(value, torch.Tensor):
|
||||
return value.to(self.dtype)
|
||||
if isinstance(value, list):
|
||||
return [_to_model_dtype(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(_to_model_dtype(item) for item in value)
|
||||
return value
|
||||
|
||||
def _to_sampling_dtype(
|
||||
prediction: torch.Tensor | tuple[torch.Tensor, ...],
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
if isinstance(prediction, tuple):
|
||||
return tuple(item.to(self.sampling_dtype) for item in prediction)
|
||||
return prediction.to(self.sampling_dtype)
|
||||
|
||||
cache_key = kwargs.pop("_cosmos3_cache_key", None)
|
||||
context_name = str(kwargs.pop("_cache_context", "cond"))
|
||||
for key in (
|
||||
"hidden_states",
|
||||
"action_latents",
|
||||
"sound_latents",
|
||||
"control_latents",
|
||||
"noisy_frame_mask",
|
||||
"action_noisy_mask",
|
||||
):
|
||||
if key in kwargs and kwargs[key] is not None:
|
||||
kwargs[key] = _to_model_dtype(kwargs[key])
|
||||
|
||||
context_factory = getattr(self, "_cache_context_factory", None)
|
||||
context = context_factory(context_name) if callable(context_factory) else nullcontext()
|
||||
@@ -1351,7 +1379,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
)
|
||||
prediction = self.transformer(**kwargs)
|
||||
branch_caches[cache_key] = (self.transformer.cached_kv, self.transformer.cached_freqs_gen)
|
||||
return prediction
|
||||
return _to_sampling_dtype(prediction)
|
||||
|
||||
def combine_multi_branch_cfg_noise(
|
||||
self,
|
||||
@@ -2131,7 +2159,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
height // self.vae_scale_factor_spatial,
|
||||
width // self.vae_scale_factor_spatial,
|
||||
)
|
||||
return randn_tensor(shape, generator=generator, device=self.device, dtype=self.dtype)
|
||||
return randn_tensor(shape, generator=generator, device=self.device, dtype=self.sampling_dtype)
|
||||
|
||||
def _prepare_sound_latents(
|
||||
self,
|
||||
@@ -2163,7 +2191,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
(1, sound_dim, latent_frames),
|
||||
generator=generator,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
dtype=self.sampling_dtype,
|
||||
)
|
||||
return latents, latent_frames
|
||||
|
||||
@@ -2400,7 +2428,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
video = image_tensor.unsqueeze(2)
|
||||
latent = self.vae.encode(video).latent_dist.mode()
|
||||
latent = self._normalize_vae_latent(latent)
|
||||
return latent[:, :, 0:1, :, :].to(self.dtype)
|
||||
return latent[:, :, 0:1, :, :].to(self.sampling_dtype)
|
||||
|
||||
def _latent_hw_from_image_size(self, image_size: Any | None) -> tuple[int, int] | None:
|
||||
if image_size is None:
|
||||
@@ -2436,7 +2464,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
latent = self.vae.encode(video).latent_dist.mode()
|
||||
latent = self._normalize_vae_latent(latent)
|
||||
latent = self._crop_latent_to_image_size(latent, image_size)
|
||||
return latent.to(self.dtype)
|
||||
return latent.to(self.sampling_dtype)
|
||||
|
||||
def _prepare_latents_i2v(
|
||||
self,
|
||||
@@ -2462,14 +2490,14 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
(1, C, T_lat, H_lat, W_lat),
|
||||
generator=generator,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
dtype=self.sampling_dtype,
|
||||
)
|
||||
|
||||
image_latent = self._encode_conditioning_image_latent(image_tensor)
|
||||
latents = noise
|
||||
latents[:, :, 0:1, :, :] = image_latent
|
||||
|
||||
velocity_mask = torch.ones(1, 1, T_lat, 1, 1, device=self.device, dtype=self.dtype)
|
||||
velocity_mask = torch.ones(1, 1, T_lat, 1, 1, device=self.device, dtype=self.sampling_dtype)
|
||||
velocity_mask[:, :, 0, :, :] = 0.0
|
||||
return latents, velocity_mask, image_latent
|
||||
|
||||
@@ -2507,7 +2535,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
(1, C, T_lat, H_lat, W_lat),
|
||||
generator=generator,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
dtype=self.sampling_dtype,
|
||||
)
|
||||
condition_pixel_frames = condition_pixel_frame_count(indexes, self.vae_scale_factor_temporal)
|
||||
condition_video = video_tensor[:, :, :condition_pixel_frames]
|
||||
@@ -2528,7 +2556,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
f"encoded={tuple(cond_prefix_latent.shape)}, expected at least {expected_prefix}."
|
||||
)
|
||||
|
||||
condition_mask = torch.zeros(1, 1, T_lat, 1, 1, device=self.device, dtype=self.dtype)
|
||||
condition_mask = torch.zeros(1, 1, T_lat, 1, 1, device=self.device, dtype=self.sampling_dtype)
|
||||
condition_latents = torch.zeros_like(noise)
|
||||
for index in indexes:
|
||||
condition_mask[:, :, index, :, :] = 1.0
|
||||
@@ -2575,7 +2603,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
(1, C, T_lat, H_lat, W_lat),
|
||||
generator=generator,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
dtype=self.sampling_dtype,
|
||||
)
|
||||
condition_indexes = vision_condition_indexes(mode, num_frames, self.vae_scale_factor_temporal)
|
||||
condition_video = video_tensor[:, :, :1] if condition_indexes == [0] else video_tensor
|
||||
@@ -2600,7 +2628,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
num_frames,
|
||||
self.vae_scale_factor_temporal,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
dtype=self.sampling_dtype,
|
||||
)
|
||||
latents = condition_mask * condition_latents + (1.0 - condition_mask) * noise
|
||||
velocity_mask = 1.0 - condition_mask
|
||||
@@ -2653,26 +2681,26 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
if raw_action_dim <= 0 or raw_action_dim > action_dim:
|
||||
raise ValueError(f"Cosmos3 raw_action_dim must be in [1, {action_dim}], got {raw_action_dim}.")
|
||||
|
||||
clean_action = clean_action.to(device=self.device, dtype=self.dtype).unsqueeze(0)
|
||||
clean_action = clean_action.to(device=self.device, dtype=self.sampling_dtype).unsqueeze(0)
|
||||
if condition_indexes is None:
|
||||
condition_mask = build_action_condition_mask(
|
||||
mode,
|
||||
action_chunk_size,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
dtype=self.sampling_dtype,
|
||||
)
|
||||
else:
|
||||
condition_mask = self._build_action_condition_mask_from_indexes(
|
||||
condition_indexes,
|
||||
action_chunk_size,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
dtype=self.sampling_dtype,
|
||||
)
|
||||
noise = randn_tensor(
|
||||
(1, action_chunk_size, action_dim),
|
||||
generator=generator,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
dtype=self.sampling_dtype,
|
||||
)
|
||||
noise[:, :, raw_action_dim:] = 0
|
||||
clean_action[:, :, raw_action_dim:] = 0
|
||||
@@ -3085,7 +3113,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
condition_latents.shape,
|
||||
generator=generator,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
dtype=self.sampling_dtype,
|
||||
)
|
||||
condition_mask = torch.zeros(
|
||||
1,
|
||||
@@ -3094,7 +3122,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
1,
|
||||
1,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
dtype=self.sampling_dtype,
|
||||
)
|
||||
if current_conditional_frames > 0:
|
||||
latent_frames = (current_conditional_frames - 1) // self.vae_scale_factor_temporal + 1
|
||||
|
||||
Reference in new issue
Block a user