feat: enhance model loading with prefetching and retry logic

- Introduced `from_pretrained_with_prefetch` to handle racy cache scenarios by re-prefetching and retrying on failures.
- Updated various model pipelines to utilize the new prefetching mechanism, ensuring robust loading of model components.
- Prefetch logic added to multiple models, including HunyuanVideo, LongCatImage, and QwenImage, to mitigate issues with incomplete cache states.
This commit is contained in:
tzhouam committed 2026-06-02 09:01:47 +00:00
1 parent b550709bba
commit 617c6ca55f
20 files changed
+678 -194

No files matched your search

@@ -26,7 +26,7 @@ from vllm_omni.diffusion.distributed.pipeline_parallel import AsyncLatents, Pipe
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.forward_context import set_forward_context_denoise_step_idx
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.model_loader.hub_prefetch import prefetch_subfolders
from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders
from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
@@ -336,18 +336,39 @@ class Wan22Pipeline(
)
# See ``hub_prefetch.py`` for the transformers v5 subfolder race.
component_subfolders = ["tokenizer", "text_encoder", "vae"]
prefetch_subfolders(
model,
["tokenizer", "text_encoder", "vae"],
component_subfolders,
local_files_only=local_files_only,
)
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
self.text_encoder = UMT5EncoderModel.from_pretrained(
model, subfolder="text_encoder", torch_dtype=dtype, local_files_only=local_files_only
# ``from_pretrained_with_prefetch`` re-prefetches and retries if the
# cache is still half-written (the missing-shard ``OSError`` and the
# default-``UMT5Config`` size-mismatch ``RuntimeError`` seen on multi
# -worker HSDP / ring launches), instead of crashing the worker.
self.tokenizer = from_pretrained_with_prefetch(
AutoTokenizer.from_pretrained,
model,
subfolder="tokenizer",
prefetch_list=component_subfolders,
local_files_only=local_files_only,
)
self.text_encoder = from_pretrained_with_prefetch(
UMT5EncoderModel.from_pretrained,
model,
subfolder="text_encoder",
prefetch_list=component_subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
).to(self.device)
self.vae = DistributedAutoencoderKLWan.from_pretrained(
model, subfolder="vae", torch_dtype=dtype, local_files_only=local_files_only
self.vae = from_pretrained_with_prefetch(
DistributedAutoencoderKLWan.from_pretrained,
model,
subfolder="vae",
prefetch_list=component_subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
).to(self.device)
# Initialize transformers with correct config (weights loaded via load_weights)
@@ -26,7 +26,7 @@ from vllm_omni.diffusion.distributed.pipeline_parallel import AsyncLatents, Pipe
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.forward_context import set_forward_context_denoise_step_idx
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.model_loader.hub_prefetch import prefetch_subfolders
from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders
from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin
from vllm_omni.diffusion.models.interface import SupportImageInput
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero
@@ -197,35 +197,64 @@ class Wan22I2VPipeline(
)
)
# Text encoder
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
self.text_encoder = UMT5EncoderModel.from_pretrained(
model, subfolder="text_encoder", torch_dtype=dtype, local_files_only=local_files_only
).to(self.device)
# Image encoder (CLIP) - optional, for Wan2.1-style I2V
self.has_image_encoder = "image_encoder" in model_index and model_index["image_encoder"][0] is not None
# See ``hub_prefetch.py`` for the transformers v5 subfolder race.
# See ``hub_prefetch.py`` for the transformers v5 subfolder race. The
# prefetch MUST run before any ``from_pretrained`` below - previously
# the tokenizer / text_encoder were loaded ahead of this call, so they
# never benefited from the prefetch and could hit the half-written
# cache directly.
subfolders = ["tokenizer", "text_encoder", "vae"]
if self.has_image_encoder:
subfolders.extend(["image_processor", "image_encoder"])
prefetch_subfolders(model, subfolders, local_files_only=local_files_only)
# Text encoder
self.tokenizer = from_pretrained_with_prefetch(
AutoTokenizer.from_pretrained,
model,
subfolder="tokenizer",
prefetch_list=subfolders,
local_files_only=local_files_only,
)
self.text_encoder = from_pretrained_with_prefetch(
UMT5EncoderModel.from_pretrained,
model,
subfolder="text_encoder",
prefetch_list=subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
).to(self.device)
if self.has_image_encoder:
self.image_processor = CLIPImageProcessor.from_pretrained(
model, subfolder="image_processor", local_files_only=local_files_only
self.image_processor = from_pretrained_with_prefetch(
CLIPImageProcessor.from_pretrained,
model,
subfolder="image_processor",
prefetch_list=subfolders,
local_files_only=local_files_only,
)
self.image_encoder = CLIPVisionModel.from_pretrained(
model, subfolder="image_encoder", torch_dtype=dtype, local_files_only=local_files_only
self.image_encoder = from_pretrained_with_prefetch(
CLIPVisionModel.from_pretrained,
model,
subfolder="image_encoder",
prefetch_list=subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
).to(self.device)
else:
self.image_processor = None
self.image_encoder = None
# VAE
self.vae = DistributedAutoencoderKLWan.from_pretrained(
model, subfolder="vae", torch_dtype=dtype, local_files_only=local_files_only
self.vae = from_pretrained_with_prefetch(
DistributedAutoencoderKLWan.from_pretrained,
model,
subfolder="vae",
prefetch_list=subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
).to(self.device)
# Transformers (weights loaded via load_weights)