feat: enhance model loading with prefetching and retry logic

- Introduced `from_pretrained_with_prefetch` to handle racy cache scenarios by re-prefetching and retrying on failures.
- Updated various model pipelines to utilize the new prefetching mechanism, ensuring robust loading of model components.
- Prefetch logic added to multiple models, including HunyuanVideo, LongCatImage, and QwenImage, to mitigate issues with incomplete cache states.
This commit is contained in:
tzhouam committed 2026-06-02 09:01:47 +00:00
1 parent b550709bba
commit 617c6ca55f
20 files changed
+678 -194

No files matched your search

@@ -36,6 +36,7 @@ from vllm_omni.diffusion.distributed.parallel_state import (
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders
from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
@@ -185,6 +186,19 @@ class LTX2Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin):
),
]
# See ``hub_prefetch.py`` for the transformers v5 multi-worker subfolder
# race; prefetch the whole component set before any from_pretrained.
ltx2_subfolders = [
"tokenizer",
"text_encoder",
"connectors",
"vae",
"audio_vae",
"vocoder",
"scheduler",
]
prefetch_subfolders(model, ltx2_subfolders, local_files_only=local_files_only)
self.tokenizer = AutoTokenizer.from_pretrained(
model,
subfolder="tokenizer",
@@ -193,36 +207,46 @@ class LTX2Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin):
# prefer mmap loading as default device is cuda, and the output of text encoder
# could be deterministic.
with torch.device("cpu"):
self.text_encoder = Gemma3ForConditionalGeneration.from_pretrained(
self.text_encoder = from_pretrained_with_prefetch(
Gemma3ForConditionalGeneration.from_pretrained,
model,
subfolder="text_encoder",
torch_dtype=dtype,
prefetch_list=ltx2_subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
).to(self.device)
self.connectors = LTX2TextConnectors.from_pretrained(
self.connectors = from_pretrained_with_prefetch(
LTX2TextConnectors.from_pretrained,
model,
subfolder="connectors",
torch_dtype=dtype,
prefetch_list=ltx2_subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
).to(self.device)
self.vae = AutoencoderKLLTX2Video.from_pretrained(
self.vae = from_pretrained_with_prefetch(
AutoencoderKLLTX2Video.from_pretrained,
model,
subfolder="vae",
torch_dtype=dtype,
prefetch_list=ltx2_subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
).to(self.device)
self.audio_vae = AutoencoderKLLTX2Audio.from_pretrained(
self.audio_vae = from_pretrained_with_prefetch(
AutoencoderKLLTX2Audio.from_pretrained,
model,
subfolder="audio_vae",
torch_dtype=dtype,
prefetch_list=ltx2_subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
).to(self.device)
self.vocoder = LTX2Vocoder.from_pretrained(
self.vocoder = from_pretrained_with_prefetch(
LTX2Vocoder.from_pretrained,
model,
subfolder="vocoder",
torch_dtype=dtype,
prefetch_list=ltx2_subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
).to(self.device)
transformer_config = load_transformer_config(model, "transformer", local_files_only)
@@ -45,6 +45,7 @@ from vllm_omni.diffusion.distributed.parallel_state import (
)
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
@@ -149,26 +150,59 @@ class LTX23Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin):
),
]
# See ``hub_prefetch.py`` for the transformers v5 multi-worker subfolder
# race; prefetch the whole component set before any from_pretrained.
ltx2_subfolders = [
"tokenizer",
"text_encoder",
"connectors",
"vae",
"audio_vae",
"vocoder",
"scheduler",
]
prefetch_subfolders(model, ltx2_subfolders, local_files_only=local_files_only)
# --- Tokenizer (lightweight, stays wherever) ---
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
# --- Text encoder: load on CPU, move to GPU only during encoding ---
with torch.device("cpu"):
self.text_encoder = Gemma3ForConditionalGeneration.from_pretrained(
model, subfolder="text_encoder", torch_dtype=dtype, local_files_only=local_files_only
self.text_encoder = from_pretrained_with_prefetch(
Gemma3ForConditionalGeneration.from_pretrained,
model,
subfolder="text_encoder",
prefetch_list=ltx2_subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
)
# --- Connectors: CPU (LTX-2.3 connectors include caption projection) ---
self.connectors = LTX2TextConnectors.from_pretrained(
model, subfolder="connectors", torch_dtype=dtype, local_files_only=local_files_only
self.connectors = from_pretrained_with_prefetch(
LTX2TextConnectors.from_pretrained,
model,
subfolder="connectors",
prefetch_list=ltx2_subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
)
# --- VAE, Audio VAE: CPU ---
self.vae = AutoencoderKLLTX2Video.from_pretrained(
model, subfolder="vae", torch_dtype=dtype, local_files_only=local_files_only
self.vae = from_pretrained_with_prefetch(
AutoencoderKLLTX2Video.from_pretrained,
model,
subfolder="vae",
prefetch_list=ltx2_subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
)
self.audio_vae = AutoencoderKLLTX2Audio.from_pretrained(
model, subfolder="audio_vae", torch_dtype=dtype, local_files_only=local_files_only
self.audio_vae = from_pretrained_with_prefetch(
AutoencoderKLLTX2Audio.from_pretrained,
model,
subfolder="audio_vae",
prefetch_list=ltx2_subfolders,
local_files_only=local_files_only,
torch_dtype=dtype,
)
# --- Vocoder: prefer BWE vocoder (48kHz) for LTX-2.3 ---