[Feat] Support Pipeline Resolution Based on HF Config & Make Pipeline Registry Eager (#3760)
Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
1 parent
f6894f4308
commit
465f1464d2
44 files changed
+1038
-1126
No files matched your search
@@ -0,0 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,3 +1,8 @@
|
||||
from .bagel import OmniBagelForConditionalGeneration
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
__all__ = ["OmniBagelForConditionalGeneration"]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm_omni.model_executor.models.dreamzero.pipeline import DREAMZERO_PIPELINE
|
||||
|
||||
__all__ = ["DREAMZERO_PIPELINE"]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,59 +1,8 @@
|
||||
from __future__ import annotations
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from .dynin_omni import DyninOmniForConditionalGeneration
|
||||
from .dynin_omni_common import (
|
||||
get_dynin_magvit_attr,
|
||||
get_dynin_modeling_attr,
|
||||
get_dynin_sampling_attr,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .dynin_omni_token2audio import DyninOmniToken2Audio
|
||||
from .dynin_omni_token2image import DyninOmniToken2Image
|
||||
from .dynin_omni_token2text import DyninOmniToken2Text
|
||||
|
||||
|
||||
_STAGE_EXPORTS = {
|
||||
"DyninOmniToken2Audio": (".dynin_omni_token2audio", "DyninOmniToken2Audio"),
|
||||
"DyninOmniToken2Image": (".dynin_omni_token2image", "DyninOmniToken2Image"),
|
||||
"DyninOmniToken2Text": (".dynin_omni_token2text", "DyninOmniToken2Text"),
|
||||
}
|
||||
|
||||
_MODELING_EXPORTS = {"DyninOmniConfig", "DyninOmniModelLM", "VideoTokenMerger"}
|
||||
_MAGVIT_EXPORTS = {"VQGANEncoder", "VQGANDecoder", "LFQuantizer", "MAGVITv2"}
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name in _STAGE_EXPORTS:
|
||||
module_name, attr_name = _STAGE_EXPORTS[name]
|
||||
module = __import__(module_name, globals(), locals(), [attr_name], 1)
|
||||
return getattr(module, attr_name)
|
||||
|
||||
if name in _MODELING_EXPORTS:
|
||||
return get_dynin_modeling_attr(name)
|
||||
|
||||
if name in _MAGVIT_EXPORTS:
|
||||
return get_dynin_magvit_attr(name)
|
||||
|
||||
if name == "get_mask_schedule":
|
||||
return get_dynin_sampling_attr("get_mask_schedule")
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DyninOmniForConditionalGeneration",
|
||||
"DyninOmniToken2Audio",
|
||||
"DyninOmniToken2Image",
|
||||
"DyninOmniToken2Text",
|
||||
"DyninOmniConfig",
|
||||
"DyninOmniModelLM",
|
||||
"VideoTokenMerger",
|
||||
"VQGANEncoder",
|
||||
"VQGANDecoder",
|
||||
"LFQuantizer",
|
||||
"MAGVITv2",
|
||||
"get_mask_schedule",
|
||||
]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,16 +1,8 @@
|
||||
def __getattr__(name: str):
|
||||
"""Lazy import to avoid importing transformers.models.glm_image at module init.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
The AR model depends on ``transformers.models.glm_image`` which is only
|
||||
available when the model weights are present. Importing it eagerly in
|
||||
``__init__.py`` breaks the pipeline registry lookup for environments that
|
||||
don't have this custom transformers extension installed.
|
||||
"""
|
||||
if name == "GlmImageForConditionalGeneration":
|
||||
from .glm_image_ar import GlmImageForConditionalGeneration
|
||||
|
||||
return GlmImageForConditionalGeneration
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
__all__ = ["GlmImageForConditionalGeneration"]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,18 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-TTS model support for vLLM-Omni.
|
||||
|
||||
GLM-TTS is a two-stage text-to-speech system:
|
||||
- Stage 0 (LLM): Llama-based AR model generates speech tokens from text
|
||||
- Stage 1 (DiT): Flow matching model converts speech tokens to mel-spectrogram
|
||||
|
||||
Reference: https://github.com/zai-org/GLM-TTS
|
||||
"""
|
||||
|
||||
from vllm_omni.model_executor.models.glm_tts.glm_tts import (
|
||||
GLMTTSForConditionalGeneration,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"GLMTTSForConditionalGeneration",
|
||||
]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,2 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,9 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vllm-omni integration for boson-ai's higgs-audio v2 (two-stage TTS)."""
|
||||
|
||||
from vllm_omni.model_executor.models.higgs_audio_v2.configuration_higgs_audio_v2 import (
|
||||
HiggsAudioV2Config,
|
||||
)
|
||||
|
||||
__all__ = ["HiggsAudioV2Config"]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,9 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vllm-omni integration for boson-ai's higgs-audio v3 (two-stage TTS)."""
|
||||
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
|
||||
__all__ = ["HiggsAudioV3Config"]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,189 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Configuration class for higgs-audio v3 (HiggsMultimodalQwen3) in vllm-omni.
|
||||
|
||||
``HiggsAudioV3Config.from_pretrained(model_path)`` returns a config with
|
||||
``tts_token_id``, ``text_token_id``, ``audio_continuation_id``, and
|
||||
``eos_token_id`` already resolved from the checkpoint tokenizer. If the
|
||||
tokenizer is unavailable or missing required specials, the load raises.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import transformers
|
||||
from transformers import AutoConfig, PretrainedConfig
|
||||
|
||||
__all__ = ["HiggsAudioV3Config"]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_QWEN3_ROPE_THETA = 1_000_000
|
||||
|
||||
_REQUIRED_SPECIALS = ("<|tts|>", "<|text|>", "<|audio|>")
|
||||
|
||||
|
||||
def _build_text_config(raw: Any) -> PretrainedConfig:
|
||||
if isinstance(raw, PretrainedConfig):
|
||||
return raw
|
||||
cfg = dict(raw or {})
|
||||
model_type = cfg.get("model_type", "qwen3")
|
||||
if model_type == "qwen3" and cfg.get("rope_theta") is None:
|
||||
cfg["rope_theta"] = _QWEN3_ROPE_THETA
|
||||
try:
|
||||
cfg_cls = transformers.CONFIG_MAPPING[model_type]
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"Unknown text backbone model_type {model_type!r}") from exc
|
||||
return cfg_cls(**cfg)
|
||||
|
||||
|
||||
def _resolve_model_dir(pretrained_model_name_or_path: str) -> str | None:
|
||||
"""Resolve a model name/path to a local directory containing tokenizer files."""
|
||||
if os.path.isdir(pretrained_model_name_or_path):
|
||||
return pretrained_model_name_or_path
|
||||
try:
|
||||
from huggingface_hub import try_to_load_from_cache
|
||||
|
||||
cached = try_to_load_from_cache(repo_id=pretrained_model_name_or_path, filename="tokenizer_config.json")
|
||||
if isinstance(cached, str) and os.path.isfile(cached):
|
||||
return os.path.dirname(cached)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
from huggingface_hub.constants import HF_HUB_CACHE
|
||||
|
||||
safe = pretrained_model_name_or_path.replace("/", "--")
|
||||
snapshots = os.path.join(HF_HUB_CACHE, f"models--{safe}", "snapshots")
|
||||
if os.path.isdir(snapshots):
|
||||
for rev in os.listdir(snapshots):
|
||||
candidate = os.path.join(snapshots, rev)
|
||||
if os.path.isfile(os.path.join(candidate, "tokenizer_config.json")):
|
||||
return candidate
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
class HiggsAudioV3Config(PretrainedConfig):
|
||||
"""Typed config for higgs-audio v3 (HiggsMultimodalQwen3).
|
||||
|
||||
``from_pretrained()`` automatically resolves ``<|tts|>``, ``<|text|>``,
|
||||
``<|audio|>`` and ``eos_token_id`` from the checkpoint tokenizer.
|
||||
"""
|
||||
|
||||
model_type: str = "higgs_multimodal_qwen3"
|
||||
is_composition = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
audio_encoder_config: dict[str, Any] | None = None,
|
||||
text_config: dict[str, Any] | PretrainedConfig | None = None,
|
||||
audio_token_id: int = -100,
|
||||
mel_per_sample: int = 8,
|
||||
num_codebooks: int = 8,
|
||||
codebook_size: int = 1026,
|
||||
audio_stream_bos_id: int = 1024,
|
||||
audio_stream_eos_id: int = 1025,
|
||||
sample_rate: int = 24000,
|
||||
frame_rate: int = 25,
|
||||
tts_token_id: int | None = None,
|
||||
text_token_id: int | None = None,
|
||||
audio_continuation_id: int | None = None,
|
||||
enable_flashinfer_api_unwrap: bool = True,
|
||||
enable_mlp_cudagraph: bool = True,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
# Legacy perf knob removed: Higgs v3 scheduler tokens now come from
|
||||
# SamplerOutput.sampled_token_ids only.
|
||||
kwargs.pop("enable_cpu_token_override", None)
|
||||
self.audio_token_id = audio_token_id
|
||||
self.mel_per_sample = mel_per_sample
|
||||
|
||||
if audio_encoder_config is None:
|
||||
audio_encoder_config = {
|
||||
"encoder_type": "discrete",
|
||||
"num_codebooks": num_codebooks,
|
||||
"vocab_size": codebook_size,
|
||||
"tie_word_embeddings": True,
|
||||
}
|
||||
self.audio_encoder_config = audio_encoder_config
|
||||
|
||||
self.num_codebooks = int(audio_encoder_config.get("num_codebooks", num_codebooks))
|
||||
if self.num_codebooks <= 0:
|
||||
raise ValueError(f"num_codebooks must be > 0, got {self.num_codebooks}")
|
||||
self.codebook_size = int(audio_encoder_config.get("vocab_size", codebook_size))
|
||||
self.tie_modality_embeddings = bool(audio_encoder_config.get("tie_word_embeddings", True))
|
||||
|
||||
self.audio_stream_bos_id = audio_stream_bos_id
|
||||
self.audio_stream_eos_id = audio_stream_eos_id
|
||||
self.sample_rate = sample_rate
|
||||
self.frame_rate = frame_rate
|
||||
|
||||
self.text_config = _build_text_config(text_config)
|
||||
self.audio_hidden_size = int(audio_encoder_config.get("out_dim", self.text_config.hidden_size))
|
||||
|
||||
self.tts_token_id = tts_token_id
|
||||
self.text_token_id = text_token_id
|
||||
self.audio_continuation_id = audio_continuation_id
|
||||
self.enable_flashinfer_api_unwrap = bool(enable_flashinfer_api_unwrap)
|
||||
self.enable_mlp_cudagraph = bool(enable_mlp_cudagraph)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs: Any) -> HiggsAudioV3Config:
|
||||
"""Load config and resolve special token IDs from the checkpoint tokenizer.
|
||||
|
||||
Passes the original ``pretrained_model_name_or_path`` (local dir or
|
||||
HF repo ID) directly to ``AutoTokenizer.from_pretrained()`` so it can
|
||||
handle cache hits, downloads, and local paths uniformly. Raises if the
|
||||
tokenizer is missing required specials.
|
||||
"""
|
||||
config = super().from_pretrained(pretrained_model_name_or_path, **kwargs)
|
||||
config.resolve_special_tokens(pretrained_model_name_or_path)
|
||||
return config
|
||||
|
||||
def get_text_config(self, decoder: bool = False) -> PretrainedConfig:
|
||||
del decoder
|
||||
return self.text_config
|
||||
|
||||
@property
|
||||
def num_real_codes(self) -> int:
|
||||
return self.audio_stream_bos_id
|
||||
|
||||
@property
|
||||
def hidden_size(self) -> int:
|
||||
return self.text_config.hidden_size
|
||||
|
||||
def resolve_special_tokens(self, model_path: str) -> None:
|
||||
"""Resolve <|tts|>, <|text|>, <|audio|> and eos from the HF tokenizer.
|
||||
|
||||
Raises ``ValueError`` if any of the 3 required specials is missing
|
||||
from the tokenizer's added vocabulary.
|
||||
"""
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
|
||||
vocab = dict(tokenizer.get_added_vocab())
|
||||
|
||||
missing = [t for t in _REQUIRED_SPECIALS if t not in vocab]
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"Tokenizer at {model_path} is missing required Higgs TTS v3 "
|
||||
f"special tokens: {missing}. Available added tokens: "
|
||||
f"{sorted(vocab.keys())[:20]}"
|
||||
)
|
||||
|
||||
self.tts_token_id = vocab["<|tts|>"]
|
||||
self.text_token_id = vocab["<|text|>"]
|
||||
self.audio_continuation_id = vocab["<|audio|>"]
|
||||
|
||||
if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None:
|
||||
self.eos_token_id = int(tokenizer.eos_token_id)
|
||||
|
||||
|
||||
AutoConfig.register("higgs_multimodal_qwen3", HiggsAudioV3Config)
|
||||
@@ -23,11 +23,11 @@ from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_decoder import (
|
||||
build_higgs_audio_acoustic_decoder,
|
||||
load_higgs_audio_codec,
|
||||
)
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
from vllm_omni.transformers_utils.configs.higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"HiggsAudioV3Code2Wav",
|
||||
|
||||
@@ -36,10 +36,10 @@ from vllm.model_executor.models.qwen3 import Qwen3Model
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.outputs import SamplerOutput
|
||||
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.transformers_utils.configs.higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
__all__ = ["HiggsAudioV3TalkerForConditionalGeneration"]
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .hunyuan_image3 import HunyuanImage3ForConditionalGeneration
|
||||
|
||||
__all__ = ["HunyuanImage3ForConditionalGeneration"]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,2 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,9 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Lance (ByteDance) — vLLM-Omni model_executor package.
|
||||
|
||||
Single-stage topology: a self-contained diffusion stage (``LancePipeline``
|
||||
defined in ``vllm_omni.diffusion.models.lance``) that owns the Qwen2-MoT LLM,
|
||||
the Qwen2.5-VL ViT, the Wan2.2 VAE and the tokenizer — directly analogous to
|
||||
``bagel_single_stage``. The ``PipelineConfig`` lives in :mod:`.pipeline`.
|
||||
"""
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,17 +1,8 @@
|
||||
from transformers import AutoTokenizer
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm_omni.tokenizers.mammoth_moda2_tokenizer import MammothUTokenizer
|
||||
from vllm_omni.transformers_utils.configs.mammoth_moda2 import ( # noqa: F401 registers AutoConfig
|
||||
Mammothmoda2Config,
|
||||
Mammothmoda2Qwen2_5_VLConfig,
|
||||
)
|
||||
|
||||
from .mammoth_moda2 import MammothModa2ARForConditionalGeneration
|
||||
|
||||
# AutoConfig.register is done inside transformers_utils/configs/mammoth_moda2.py
|
||||
AutoTokenizer.register(config_class=Mammothmoda2Config, slow_tokenizer_class=MammothUTokenizer)
|
||||
AutoTokenizer.register(config_class=Mammothmoda2Qwen2_5_VLConfig, slow_tokenizer_class=MammothUTokenizer)
|
||||
|
||||
__all__ = [
|
||||
"MammothModa2ARForConditionalGeneration",
|
||||
]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,20 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .ming_flash_omni import MingFlashOmniForConditionalGeneration
|
||||
from .ming_flash_omni_talker import MingFlashOmniTalkerForConditionalGeneration
|
||||
from .ming_flash_omni_thinker import (
|
||||
MingFlashOmniThinkerDummyInputsBuilder,
|
||||
MingFlashOmniThinkerForConditionalGeneration,
|
||||
MingFlashOmniThinkerMultiModalProcessor,
|
||||
MingFlashOmniThinkerProcessingInfo,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MingFlashOmniForConditionalGeneration",
|
||||
"MingFlashOmniTalkerForConditionalGeneration",
|
||||
"MingFlashOmniThinkerForConditionalGeneration",
|
||||
"MingFlashOmniThinkerProcessingInfo",
|
||||
"MingFlashOmniThinkerMultiModalProcessor",
|
||||
"MingFlashOmniThinkerDummyInputsBuilder",
|
||||
]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,13 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .config_ming_tts import MingDenseConfig
|
||||
from .ming_tts import MingTTSForConditionalGeneration
|
||||
from .ming_tts_audio_vae import MingAudioVAEModel
|
||||
from .ming_tts_llm import MingLLMModel
|
||||
|
||||
__all__ = [
|
||||
"MingDenseConfig",
|
||||
"MingTTSForConditionalGeneration",
|
||||
"MingLLMModel",
|
||||
"MingAudioVAEModel",
|
||||
]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,9 +1,8 @@
|
||||
from .minicpmo_4_5_omni import MiniCPMO45OmniForConditionalGeneration
|
||||
from .minicpmo_4_5_omni_llm import MiniCPMO45OmniLLMForConditionalGeneration
|
||||
from .minicpmo_4_5_omni_tts import MiniCPMO45OmniTTSForConditionalGeneration
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
__all__ = [
|
||||
"MiniCPMO45OmniForConditionalGeneration",
|
||||
"MiniCPMO45OmniLLMForConditionalGeneration",
|
||||
"MiniCPMO45OmniTTSForConditionalGeneration",
|
||||
]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,13 +1,8 @@
|
||||
from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_codec import (
|
||||
MossTTSCodecDecoder,
|
||||
)
|
||||
from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_talker import (
|
||||
MossTTSDelayTalkerForGeneration,
|
||||
MossTTSRealtimeTalkerForGeneration,
|
||||
)
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
__all__ = [
|
||||
"MossTTSDelayTalkerForGeneration",
|
||||
"MossTTSRealtimeTalkerForGeneration",
|
||||
"MossTTSCodecDecoder",
|
||||
]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,5 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .modeling_moss_tts_nano import MossTTSNanoForGeneration
|
||||
|
||||
__all__ = ["MossTTSNanoForGeneration"]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -1,3 +1,8 @@
|
||||
from .qwen3_omni import Qwen3OmniMoeForConditionalGeneration
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
__all__ = ["Qwen3OmniMoeForConditionalGeneration"]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
@@ -7,10 +7,13 @@ Stage 1: Talker — text embeddings → RVQ codec codes
|
||||
Stage 2: Code2Wav — RVQ codes → audio waveform
|
||||
"""
|
||||
|
||||
from transformers import Qwen3OmniMoeConfig
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
pipeline_cfg_resolver,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.qwen3_omni"
|
||||
@@ -64,3 +67,38 @@ QWEN3_OMNI_PIPELINE = PipelineConfig(
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
QWEN3_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig(
|
||||
model_type="qwen3_omni_moe_thinker_only",
|
||||
model_arch="Qwen3OmniMoeForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="thinker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
hf_config_name="thinker_config",
|
||||
engine_output_type="latent",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pipeline_cfg_resolver(config_type=Qwen3OmniMoeConfig)
|
||||
def resolve_qwen3_omni_pipeline(
|
||||
hf_config: Qwen3OmniMoeConfig,
|
||||
) -> PipelineConfig:
|
||||
"""Select the right pipeline variant based on the HF config, since some variants,
|
||||
e.g., Qwen3-Omni-30B-A3B-Captioner, are thinker only.
|
||||
|
||||
By default, we load the full pipeline, as this is the common case.
|
||||
"""
|
||||
# If we have a config and it explicitly disabled audio input, load thinker only
|
||||
if not hf_config.enable_audio_output:
|
||||
return QWEN3_OMNI_THINKER_ONLY_PIPELINE
|
||||
return QWEN3_OMNI_PIPELINE
|
||||
@@ -106,9 +106,6 @@ class Qwen3OmniMoeTalkerForConditionalGeneration(
|
||||
self.hidden_projection = Qwen3OmniMoeTalkerResizeMLP(self.config)
|
||||
self.codec_head = nn.Linear(self.config.text_config.hidden_size, self.config.text_config.vocab_size, bias=False)
|
||||
|
||||
self.rope_deltas = None
|
||||
self.spatial_merge_size = self.config.spatial_merge_size
|
||||
|
||||
self.vocab_size = self.config.code_predictor_config.vocab_size
|
||||
self.num_code_groups = self.config.code_predictor_config.num_code_groups
|
||||
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .voxcpm2_talker import VoxCPM2TalkerForConditionalGeneration
|
||||
|
||||
__all__ = ["VoxCPM2TalkerForConditionalGeneration"]
|
||||
# NOTE: Do not import model classes in this file. Importing any
|
||||
# submodule in this package triggers __init__.py execution, and
|
||||
# both the model registry and pipeline registry import submodules
|
||||
# directly — heavy imports here would be loaded as a side effect
|
||||
# even though nothing depends on these re-exports.
|
||||
Reference in new issue
Block a user