[Core] Cleanup DiffusersPipelineLoader (#1932)

Signed-off-by: Alex Brooks <albrooks@redhat.com>

tests/diffusion/test_worker_wrapper_base.py

@@ -41,7 +41,7 @@ def mock_od_config(mocker: MockerFixture):
     config.cache_backend = None
     config.cache_config = None
     config.model = "test-model"
-    config.diffusion_load_format = None
+    config.diffusion_load_format = "default"
     config.dtype = "float32"
     config.max_cpu_loras = 0
     config.lora_path = None