[Rebase] Rebase to vllm v0.19.0 (#2475)

This commit is contained in:
Zhou Taichang authored and GitHub committed 2026-04-04 15:03:32 +08:00
1 parent 4c031580cf
commit 2804a85e58
52 files changed
+437 -142

No files matched your search

+47 -1
View File
@@ -1,6 +1,8 @@
import argparse
import dataclasses
import json
import os
import tempfile
from dataclasses import dataclass, field
from typing import Any
@@ -135,6 +137,30 @@ class OmniEngineArgs(EngineArgs):
self._omni_models_registered = True
return True
def _patch_empty_hf_config(self, model_type: str) -> None:
"""For models with empty config.json (e.g. CosyVoice3), create a
patched config in a temp directory with model_type set so that
transformers AutoConfig.from_pretrained can resolve the config class.
Sets self.hf_config_path to point to the patched directory."""
try:
from transformers import PretrainedConfig
config_dict, _ = PretrainedConfig.get_config_dict(self.model)
if config_dict.get("model_type"):
return # config.json already has model_type, no patching needed
except Exception:
return # can't load config, let vLLM handle the error
# Create a temp dir with a patched config.json
temp_dir = tempfile.mkdtemp(prefix="omni_hf_config_")
config_dict["model_type"] = model_type
config_dict.setdefault("architectures", [self.model_arch])
with open(os.path.join(temp_dir, "config.json"), "w") as f:
json.dump(config_dict, f)
self.hf_config_path = temp_dir
self._temp_config_dir = temp_dir
logger.info("Patched empty HF config with model_type=%s at %s", model_type, temp_dir)
def create_model_config(self) -> OmniModelConfig:
"""Create an OmniModelConfig from these engine arguments.
Returns:
@@ -164,6 +190,18 @@ class OmniEngineArgs(EngineArgs):
if model_type is not None:
self.hf_overrides.setdefault("model_type", model_type)
# For models whose HF config.json is empty or lacks model_type
# (e.g. CosyVoice3), AutoConfig.from_pretrained fails because it
# cannot determine which config class to use from the empty dict.
# hf_overrides alone is not enough since transformers reads
# model_type from config_dict before applying overrides.
# Workaround: create a patched config.json in a temp directory
# and point hf_config_path to it so vLLM reads model_type from it.
if not self.hf_config_path:
model_type = _ARCH_TO_MODEL_TYPE.get(self.model_arch)
if model_type is not None:
self._patch_empty_hf_config(model_type)
# Auto-detect tokenizer for models that store it in a subdirectory
# rather than the root (e.g. CosyVoice3 uses CosyVoice-BlankEN/).
if not self.tokenizer and self.model:
@@ -200,7 +238,15 @@ class OmniEngineArgs(EngineArgs):
logger.warning("Failed to download tokenizer subfolder: %s", e)
# Build the vLLM config first, then use it to create the Omni config.
model_config = super().create_model_config()
try:
model_config = super().create_model_config()
finally:
# Clean up temp config dir if we created one
if hasattr(self, "_temp_config_dir"):
import shutil
shutil.rmtree(self._temp_config_dir, ignore_errors=True)
del self._temp_config_dir
omni_config = OmniModelConfig.from_vllm_model_config(
model_config=model_config,