From 27a6321a6311870fe68c2c9c13b694f2612ff779 Mon Sep 17 00:00:00 2001 From: yzhautouskay Date: Wed, 30 Sep 2026 20:35:51 +0200 Subject: [PATCH] [Model][Cosmos3] Keep diffusion sampling state in FP32 (#7592) Signed-off-by: Yuliya Zhautouskaya --- .../models/cosmos3/test_cosmos3_pipeline.py | 52 ++++++++++++++++ .../models/cosmos3/pipeline_cosmos3.py | 62 ++++++++++++++----- 2 files changed, 97 insertions(+), 17 deletions(-) diff --git a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py index 52762cbf7..be9f753a2 100644 --- a/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py +++ b/tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py @@ -78,6 +78,12 @@ def test_component_selective_model_offload_fails_before_component_loading(monkey pipeline_module.Cosmos3OmniDiffusersPipeline(od_config=config) +def test_sampling_dtype_defaults_to_float32() -> None: + from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline + + assert Cosmos3OmniDiffusersPipeline.sampling_dtype == torch.float32 + + class StubScheduler: def __init__( self, @@ -228,6 +234,7 @@ class StubCosmos3Transformer(nn.Module): { "token": token, "has_control": control_latents is not None, + "hidden_states_dtype": hidden_states.dtype, "timestep": timestep.clone(), "text_mask": text_mask.clone(), "cache_before": self.cached_kv, @@ -2149,6 +2156,43 @@ def test_prepare_latents_for_video_image_sound_and_action(make_cosmos3_pipeline) torch.testing.assert_close(action, clean) +def test_sampling_state_casts_transformer_execution_to_model_dtype(make_cosmos3_pipeline) -> None: + pipeline = make_cosmos3_pipeline() + pipeline.dtype = torch.bfloat16 + + latents = pipeline._prepare_latents(16, 24, 5, torch.Generator(device="cpu").manual_seed(0)) + assert pipeline.sampling_dtype == torch.float32 + assert latents.dtype == torch.float32 + + prediction = pipeline.predict_noise( + hidden_states=latents, + timestep=torch.tensor([1]), + text_ids=_ids(2), + text_mask=_mask(), + ) + + assert pipeline.transformer.calls[-1]["hidden_states_dtype"] == torch.bfloat16 + assert prediction.dtype == torch.float32 + + +def test_sampling_dtype_can_use_model_dtype(make_cosmos3_pipeline) -> None: + pipeline = make_cosmos3_pipeline() + pipeline.dtype = torch.bfloat16 + pipeline.sampling_dtype = pipeline.dtype + + latents = pipeline._prepare_latents(16, 24, 5, torch.Generator(device="cpu").manual_seed(0)) + prediction = pipeline.predict_noise( + hidden_states=latents, + timestep=torch.tensor([1]), + text_ids=_ids(2), + text_mask=_mask(), + ) + + assert latents.dtype == torch.bfloat16 + assert pipeline.transformer.calls[-1]["hidden_states_dtype"] == torch.bfloat16 + assert prediction.dtype == torch.bfloat16 + + def test_prepare_latents_i2v_encodes_only_conditioning_frame(make_cosmos3_pipeline) -> None: pipeline = make_cosmos3_pipeline() calls: list[tuple[str, tuple[int, ...]]] = [] @@ -2232,6 +2276,7 @@ def test_prepare_inverse_dynamics_latents_encodes_full_video(make_cosmos3_pipeli def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> None: pipeline = make_cosmos3_pipeline() + pipeline.dtype = torch.bfloat16 latents = torch.zeros(1, 2, 1, 1, 1) result = pipeline.diffuse( @@ -2246,6 +2291,8 @@ def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> N guidance_interval=(500.0, 1000.0), ) assert [call["token"] for call in pipeline.transformer.calls] == [2, 1, 2] + assert all(call["hidden_states_dtype"] == torch.bfloat16 for call in pipeline.transformer.calls) + assert result.dtype == torch.float32 torch.testing.assert_close(result, torch.full_like(latents, 6.0)) i2v = pipeline.diffuse( @@ -2260,6 +2307,8 @@ def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> N velocity_mask=torch.tensor([[[[[0.0]], [[1.0]]]]]), image_latent=torch.full((1, 2, 1, 1, 1), 7.0), ) + assert pipeline.transformer.calls[-1]["hidden_states_dtype"] == torch.bfloat16 + assert i2v.dtype == torch.float32 torch.testing.assert_close(i2v[:, :, 0:1], torch.full((1, 2, 1, 1, 1), 7.0)) i2v_noise = pipeline.scheduler.step_calls[-1][0] torch.testing.assert_close(i2v_noise[:, :, 0:1], torch.zeros(1, 2, 1, 1, 1)) @@ -2279,6 +2328,9 @@ def test_diffuse_covers_cfg_i2v_and_multimodal_steps(make_cosmos3_pipeline) -> N guidance_scale=1.0, shared_kwargs={"video_shape": (1, 1, 1), "fps": 24.0, "action_domain_ids": torch.tensor([0])}, ) + assert all(call["hidden_states_dtype"] == torch.bfloat16 for call in pipeline.transformer.calls) + assert video_result.dtype == torch.float32 + assert action_result.dtype == torch.float32 torch.testing.assert_close(video_result, torch.full_like(latents, 4.0)) torch.testing.assert_close(action_result, torch.full((), 44.0).expand_as(action_result)) diff --git a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py index 7a7de7d70..18bf4b171 100644 --- a/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py +++ b/vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py @@ -932,6 +932,7 @@ class Cosmos3OmniDiffusersPipeline( _encoder_modules: ClassVar[list[str]] = [] _vae_modules: ClassVar[list[str]] = ["vae"] _resident_modules: ClassVar[list[str]] = [] + sampling_dtype: ClassVar[torch.dtype] = torch.float32 @classmethod def reference_video_decode_spec( @@ -1332,8 +1333,35 @@ class Cosmos3OmniDiffusersPipeline( The transformer returns the raw prediction: video-only as a tensor, or a tuple in video, action, sound order for multimodal generation. """ + + def _to_model_dtype(value: Any) -> Any: + if isinstance(value, torch.Tensor): + return value.to(self.dtype) + if isinstance(value, list): + return [_to_model_dtype(item) for item in value] + if isinstance(value, tuple): + return tuple(_to_model_dtype(item) for item in value) + return value + + def _to_sampling_dtype( + prediction: torch.Tensor | tuple[torch.Tensor, ...], + ) -> torch.Tensor | tuple[torch.Tensor, ...]: + if isinstance(prediction, tuple): + return tuple(item.to(self.sampling_dtype) for item in prediction) + return prediction.to(self.sampling_dtype) + cache_key = kwargs.pop("_cosmos3_cache_key", None) context_name = str(kwargs.pop("_cache_context", "cond")) + for key in ( + "hidden_states", + "action_latents", + "sound_latents", + "control_latents", + "noisy_frame_mask", + "action_noisy_mask", + ): + if key in kwargs and kwargs[key] is not None: + kwargs[key] = _to_model_dtype(kwargs[key]) context_factory = getattr(self, "_cache_context_factory", None) context = context_factory(context_name) if callable(context_factory) else nullcontext() @@ -1351,7 +1379,7 @@ class Cosmos3OmniDiffusersPipeline( ) prediction = self.transformer(**kwargs) branch_caches[cache_key] = (self.transformer.cached_kv, self.transformer.cached_freqs_gen) - return prediction + return _to_sampling_dtype(prediction) def combine_multi_branch_cfg_noise( self, @@ -2131,7 +2159,7 @@ class Cosmos3OmniDiffusersPipeline( height // self.vae_scale_factor_spatial, width // self.vae_scale_factor_spatial, ) - return randn_tensor(shape, generator=generator, device=self.device, dtype=self.dtype) + return randn_tensor(shape, generator=generator, device=self.device, dtype=self.sampling_dtype) def _prepare_sound_latents( self, @@ -2163,7 +2191,7 @@ class Cosmos3OmniDiffusersPipeline( (1, sound_dim, latent_frames), generator=generator, device=self.device, - dtype=self.dtype, + dtype=self.sampling_dtype, ) return latents, latent_frames @@ -2400,7 +2428,7 @@ class Cosmos3OmniDiffusersPipeline( video = image_tensor.unsqueeze(2) latent = self.vae.encode(video).latent_dist.mode() latent = self._normalize_vae_latent(latent) - return latent[:, :, 0:1, :, :].to(self.dtype) + return latent[:, :, 0:1, :, :].to(self.sampling_dtype) def _latent_hw_from_image_size(self, image_size: Any | None) -> tuple[int, int] | None: if image_size is None: @@ -2436,7 +2464,7 @@ class Cosmos3OmniDiffusersPipeline( latent = self.vae.encode(video).latent_dist.mode() latent = self._normalize_vae_latent(latent) latent = self._crop_latent_to_image_size(latent, image_size) - return latent.to(self.dtype) + return latent.to(self.sampling_dtype) def _prepare_latents_i2v( self, @@ -2462,14 +2490,14 @@ class Cosmos3OmniDiffusersPipeline( (1, C, T_lat, H_lat, W_lat), generator=generator, device=self.device, - dtype=self.dtype, + dtype=self.sampling_dtype, ) image_latent = self._encode_conditioning_image_latent(image_tensor) latents = noise latents[:, :, 0:1, :, :] = image_latent - velocity_mask = torch.ones(1, 1, T_lat, 1, 1, device=self.device, dtype=self.dtype) + velocity_mask = torch.ones(1, 1, T_lat, 1, 1, device=self.device, dtype=self.sampling_dtype) velocity_mask[:, :, 0, :, :] = 0.0 return latents, velocity_mask, image_latent @@ -2507,7 +2535,7 @@ class Cosmos3OmniDiffusersPipeline( (1, C, T_lat, H_lat, W_lat), generator=generator, device=self.device, - dtype=self.dtype, + dtype=self.sampling_dtype, ) condition_pixel_frames = condition_pixel_frame_count(indexes, self.vae_scale_factor_temporal) condition_video = video_tensor[:, :, :condition_pixel_frames] @@ -2528,7 +2556,7 @@ class Cosmos3OmniDiffusersPipeline( f"encoded={tuple(cond_prefix_latent.shape)}, expected at least {expected_prefix}." ) - condition_mask = torch.zeros(1, 1, T_lat, 1, 1, device=self.device, dtype=self.dtype) + condition_mask = torch.zeros(1, 1, T_lat, 1, 1, device=self.device, dtype=self.sampling_dtype) condition_latents = torch.zeros_like(noise) for index in indexes: condition_mask[:, :, index, :, :] = 1.0 @@ -2575,7 +2603,7 @@ class Cosmos3OmniDiffusersPipeline( (1, C, T_lat, H_lat, W_lat), generator=generator, device=self.device, - dtype=self.dtype, + dtype=self.sampling_dtype, ) condition_indexes = vision_condition_indexes(mode, num_frames, self.vae_scale_factor_temporal) condition_video = video_tensor[:, :, :1] if condition_indexes == [0] else video_tensor @@ -2600,7 +2628,7 @@ class Cosmos3OmniDiffusersPipeline( num_frames, self.vae_scale_factor_temporal, device=self.device, - dtype=self.dtype, + dtype=self.sampling_dtype, ) latents = condition_mask * condition_latents + (1.0 - condition_mask) * noise velocity_mask = 1.0 - condition_mask @@ -2653,26 +2681,26 @@ class Cosmos3OmniDiffusersPipeline( if raw_action_dim <= 0 or raw_action_dim > action_dim: raise ValueError(f"Cosmos3 raw_action_dim must be in [1, {action_dim}], got {raw_action_dim}.") - clean_action = clean_action.to(device=self.device, dtype=self.dtype).unsqueeze(0) + clean_action = clean_action.to(device=self.device, dtype=self.sampling_dtype).unsqueeze(0) if condition_indexes is None: condition_mask = build_action_condition_mask( mode, action_chunk_size, device=self.device, - dtype=self.dtype, + dtype=self.sampling_dtype, ) else: condition_mask = self._build_action_condition_mask_from_indexes( condition_indexes, action_chunk_size, device=self.device, - dtype=self.dtype, + dtype=self.sampling_dtype, ) noise = randn_tensor( (1, action_chunk_size, action_dim), generator=generator, device=self.device, - dtype=self.dtype, + dtype=self.sampling_dtype, ) noise[:, :, raw_action_dim:] = 0 clean_action[:, :, raw_action_dim:] = 0 @@ -3085,7 +3113,7 @@ class Cosmos3OmniDiffusersPipeline( condition_latents.shape, generator=generator, device=self.device, - dtype=self.dtype, + dtype=self.sampling_dtype, ) condition_mask = torch.zeros( 1, @@ -3094,7 +3122,7 @@ class Cosmos3OmniDiffusersPipeline( 1, 1, device=self.device, - dtype=self.dtype, + dtype=self.sampling_dtype, ) if current_conditional_frames > 0: latent_frames = (current_conditional_frames - 1) // self.vae_scale_factor_temporal + 1