feat: enhance model loading with prefetching and retry logic
- Introduced `from_pretrained_with_prefetch` to handle racy cache scenarios by re-prefetching and retrying on failures. - Updated various model pipelines to utilize the new prefetching mechanism, ensuring robust loading of model components. - Prefetch logic added to multiple models, including HunyuanVideo, LongCatImage, and QwenImage, to mitigate issues with incomplete cache states.
This commit is contained in:
1 parent
b550709bba
commit
617c6ca55f
20 files changed
+678
-194
No files matched your search
@@ -36,6 +36,7 @@ from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders
|
||||
from vllm_omni.diffusion.models.dmd2 import DMD2PipelineMixin
|
||||
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
@@ -185,6 +186,19 @@ class LTX2Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin):
|
||||
),
|
||||
]
|
||||
|
||||
# See ``hub_prefetch.py`` for the transformers v5 multi-worker subfolder
|
||||
# race; prefetch the whole component set before any from_pretrained.
|
||||
ltx2_subfolders = [
|
||||
"tokenizer",
|
||||
"text_encoder",
|
||||
"connectors",
|
||||
"vae",
|
||||
"audio_vae",
|
||||
"vocoder",
|
||||
"scheduler",
|
||||
]
|
||||
prefetch_subfolders(model, ltx2_subfolders, local_files_only=local_files_only)
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
model,
|
||||
subfolder="tokenizer",
|
||||
@@ -193,36 +207,46 @@ class LTX2Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin):
|
||||
# prefer mmap loading as default device is cuda, and the output of text encoder
|
||||
# could be deterministic.
|
||||
with torch.device("cpu"):
|
||||
self.text_encoder = Gemma3ForConditionalGeneration.from_pretrained(
|
||||
self.text_encoder = from_pretrained_with_prefetch(
|
||||
Gemma3ForConditionalGeneration.from_pretrained,
|
||||
model,
|
||||
subfolder="text_encoder",
|
||||
torch_dtype=dtype,
|
||||
prefetch_list=ltx2_subfolders,
|
||||
local_files_only=local_files_only,
|
||||
torch_dtype=dtype,
|
||||
).to(self.device)
|
||||
self.connectors = LTX2TextConnectors.from_pretrained(
|
||||
self.connectors = from_pretrained_with_prefetch(
|
||||
LTX2TextConnectors.from_pretrained,
|
||||
model,
|
||||
subfolder="connectors",
|
||||
torch_dtype=dtype,
|
||||
prefetch_list=ltx2_subfolders,
|
||||
local_files_only=local_files_only,
|
||||
torch_dtype=dtype,
|
||||
).to(self.device)
|
||||
|
||||
self.vae = AutoencoderKLLTX2Video.from_pretrained(
|
||||
self.vae = from_pretrained_with_prefetch(
|
||||
AutoencoderKLLTX2Video.from_pretrained,
|
||||
model,
|
||||
subfolder="vae",
|
||||
torch_dtype=dtype,
|
||||
prefetch_list=ltx2_subfolders,
|
||||
local_files_only=local_files_only,
|
||||
torch_dtype=dtype,
|
||||
).to(self.device)
|
||||
self.audio_vae = AutoencoderKLLTX2Audio.from_pretrained(
|
||||
self.audio_vae = from_pretrained_with_prefetch(
|
||||
AutoencoderKLLTX2Audio.from_pretrained,
|
||||
model,
|
||||
subfolder="audio_vae",
|
||||
torch_dtype=dtype,
|
||||
prefetch_list=ltx2_subfolders,
|
||||
local_files_only=local_files_only,
|
||||
torch_dtype=dtype,
|
||||
).to(self.device)
|
||||
self.vocoder = LTX2Vocoder.from_pretrained(
|
||||
self.vocoder = from_pretrained_with_prefetch(
|
||||
LTX2Vocoder.from_pretrained,
|
||||
model,
|
||||
subfolder="vocoder",
|
||||
torch_dtype=dtype,
|
||||
prefetch_list=ltx2_subfolders,
|
||||
local_files_only=local_files_only,
|
||||
torch_dtype=dtype,
|
||||
).to(self.device)
|
||||
|
||||
transformer_config = load_transformer_config(model, "transformer", local_files_only)
|
||||
|
||||
@@ -45,6 +45,7 @@ from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.model_loader.hub_prefetch import from_pretrained_with_prefetch, prefetch_subfolders
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
|
||||
@@ -149,26 +150,59 @@ class LTX23Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin):
|
||||
),
|
||||
]
|
||||
|
||||
# See ``hub_prefetch.py`` for the transformers v5 multi-worker subfolder
|
||||
# race; prefetch the whole component set before any from_pretrained.
|
||||
ltx2_subfolders = [
|
||||
"tokenizer",
|
||||
"text_encoder",
|
||||
"connectors",
|
||||
"vae",
|
||||
"audio_vae",
|
||||
"vocoder",
|
||||
"scheduler",
|
||||
]
|
||||
prefetch_subfolders(model, ltx2_subfolders, local_files_only=local_files_only)
|
||||
|
||||
# --- Tokenizer (lightweight, stays wherever) ---
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
|
||||
|
||||
# --- Text encoder: load on CPU, move to GPU only during encoding ---
|
||||
with torch.device("cpu"):
|
||||
self.text_encoder = Gemma3ForConditionalGeneration.from_pretrained(
|
||||
model, subfolder="text_encoder", torch_dtype=dtype, local_files_only=local_files_only
|
||||
self.text_encoder = from_pretrained_with_prefetch(
|
||||
Gemma3ForConditionalGeneration.from_pretrained,
|
||||
model,
|
||||
subfolder="text_encoder",
|
||||
prefetch_list=ltx2_subfolders,
|
||||
local_files_only=local_files_only,
|
||||
torch_dtype=dtype,
|
||||
)
|
||||
|
||||
# --- Connectors: CPU (LTX-2.3 connectors include caption projection) ---
|
||||
self.connectors = LTX2TextConnectors.from_pretrained(
|
||||
model, subfolder="connectors", torch_dtype=dtype, local_files_only=local_files_only
|
||||
self.connectors = from_pretrained_with_prefetch(
|
||||
LTX2TextConnectors.from_pretrained,
|
||||
model,
|
||||
subfolder="connectors",
|
||||
prefetch_list=ltx2_subfolders,
|
||||
local_files_only=local_files_only,
|
||||
torch_dtype=dtype,
|
||||
)
|
||||
|
||||
# --- VAE, Audio VAE: CPU ---
|
||||
self.vae = AutoencoderKLLTX2Video.from_pretrained(
|
||||
model, subfolder="vae", torch_dtype=dtype, local_files_only=local_files_only
|
||||
self.vae = from_pretrained_with_prefetch(
|
||||
AutoencoderKLLTX2Video.from_pretrained,
|
||||
model,
|
||||
subfolder="vae",
|
||||
prefetch_list=ltx2_subfolders,
|
||||
local_files_only=local_files_only,
|
||||
torch_dtype=dtype,
|
||||
)
|
||||
self.audio_vae = AutoencoderKLLTX2Audio.from_pretrained(
|
||||
model, subfolder="audio_vae", torch_dtype=dtype, local_files_only=local_files_only
|
||||
self.audio_vae = from_pretrained_with_prefetch(
|
||||
AutoencoderKLLTX2Audio.from_pretrained,
|
||||
model,
|
||||
subfolder="audio_vae",
|
||||
prefetch_list=ltx2_subfolders,
|
||||
local_files_only=local_files_only,
|
||||
torch_dtype=dtype,
|
||||
)
|
||||
|
||||
# --- Vocoder: prefer BWE vocoder (48kHz) for LTX-2.3 ---
|
||||
|
||||
Reference in new issue
Block a user