diff --git a/vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py b/vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py
index 0bb8a48cd..ac7231fbc 100644
--- a/vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py
+++ b/vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py
@@ -98,7 +98,7 @@ class StableDiffusionXLPipeline(
             prefetch_list=sdxl_subfolders,
             local_files_only=local_files_only,
             torch_dtype=dtype,
-        )
+        ).to(self.device)
         self.text_encoder_2 = from_pretrained_with_prefetch(
             CLIPTextModelWithProjection.from_pretrained,
             model,
@@ -106,7 +106,7 @@ class StableDiffusionXLPipeline(
             prefetch_list=sdxl_subfolders,
             local_files_only=local_files_only,
             torch_dtype=dtype,
-        )
+        ).to(self.device)
         self.unet = SDXLUNet2DConditionModel(od_config=od_config)
         self.vae = from_pretrained_with_prefetch(
             DistributedAutoencoderKL.from_pretrained,
@@ -146,7 +146,7 @@ class StableDiffusionXLPipeline(
         )
         text_input_ids = text_inputs.input_ids
 
-        text_encoder_device = next(text_encoder.parameters()).device
+        text_encoder_device = self.device
         outputs = text_encoder(text_input_ids.to(text_encoder_device), output_hidden_states=True)
         prompt_embeds = outputs.hidden_states[-2].to(dtype=self.od_config.dtype, device=self.device)