[Feat] Support Pipeline Resolution Based on HF Config & Make Pipeline Registry Eager (#3760)

Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
Alex Brooks authored and GitHub committed 2026-06-24 11:02:29 +08:00
1 parent f6894f4308
commit 465f1464d2
44 files changed
+1038 -1126

No files matched your search

@@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,3 +1,8 @@
from .bagel import OmniBagelForConditionalGeneration
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
__all__ = ["OmniBagelForConditionalGeneration"]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm_omni.model_executor.models.dreamzero.pipeline import DREAMZERO_PIPELINE
__all__ = ["DREAMZERO_PIPELINE"]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,59 +1,8 @@
from __future__ import annotations
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any
from .dynin_omni import DyninOmniForConditionalGeneration
from .dynin_omni_common import (
get_dynin_magvit_attr,
get_dynin_modeling_attr,
get_dynin_sampling_attr,
)
if TYPE_CHECKING:
from .dynin_omni_token2audio import DyninOmniToken2Audio
from .dynin_omni_token2image import DyninOmniToken2Image
from .dynin_omni_token2text import DyninOmniToken2Text
_STAGE_EXPORTS = {
"DyninOmniToken2Audio": (".dynin_omni_token2audio", "DyninOmniToken2Audio"),
"DyninOmniToken2Image": (".dynin_omni_token2image", "DyninOmniToken2Image"),
"DyninOmniToken2Text": (".dynin_omni_token2text", "DyninOmniToken2Text"),
}
_MODELING_EXPORTS = {"DyninOmniConfig", "DyninOmniModelLM", "VideoTokenMerger"}
_MAGVIT_EXPORTS = {"VQGANEncoder", "VQGANDecoder", "LFQuantizer", "MAGVITv2"}
def __getattr__(name: str) -> Any:
if name in _STAGE_EXPORTS:
module_name, attr_name = _STAGE_EXPORTS[name]
module = __import__(module_name, globals(), locals(), [attr_name], 1)
return getattr(module, attr_name)
if name in _MODELING_EXPORTS:
return get_dynin_modeling_attr(name)
if name in _MAGVIT_EXPORTS:
return get_dynin_magvit_attr(name)
if name == "get_mask_schedule":
return get_dynin_sampling_attr("get_mask_schedule")
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
__all__ = [
"DyninOmniForConditionalGeneration",
"DyninOmniToken2Audio",
"DyninOmniToken2Image",
"DyninOmniToken2Text",
"DyninOmniConfig",
"DyninOmniModelLM",
"VideoTokenMerger",
"VQGANEncoder",
"VQGANDecoder",
"LFQuantizer",
"MAGVITv2",
"get_mask_schedule",
]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,16 +1,8 @@
def __getattr__(name: str):
"""Lazy import to avoid importing transformers.models.glm_image at module init.
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
The AR model depends on ``transformers.models.glm_image`` which is only
available when the model weights are present. Importing it eagerly in
``__init__.py`` breaks the pipeline registry lookup for environments that
don't have this custom transformers extension installed.
"""
if name == "GlmImageForConditionalGeneration":
from .glm_image_ar import GlmImageForConditionalGeneration
return GlmImageForConditionalGeneration
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
__all__ = ["GlmImageForConditionalGeneration"]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,18 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-TTS model support for vLLM-Omni.
GLM-TTS is a two-stage text-to-speech system:
- Stage 0 (LLM): Llama-based AR model generates speech tokens from text
- Stage 1 (DiT): Flow matching model converts speech tokens to mel-spectrogram
Reference: https://github.com/zai-org/GLM-TTS
"""
from vllm_omni.model_executor.models.glm_tts.glm_tts import (
GLMTTSForConditionalGeneration,
)
__all__ = [
"GLMTTSForConditionalGeneration",
]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,2 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,9 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vllm-omni integration for boson-ai's higgs-audio v2 (two-stage TTS)."""
from vllm_omni.model_executor.models.higgs_audio_v2.configuration_higgs_audio_v2 import (
HiggsAudioV2Config,
)
__all__ = ["HiggsAudioV2Config"]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,9 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vllm-omni integration for boson-ai's higgs-audio v3 (two-stage TTS)."""
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
__all__ = ["HiggsAudioV3Config"]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,189 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration class for higgs-audio v3 (HiggsMultimodalQwen3) in vllm-omni.
``HiggsAudioV3Config.from_pretrained(model_path)`` returns a config with
``tts_token_id``, ``text_token_id``, ``audio_continuation_id``, and
``eos_token_id`` already resolved from the checkpoint tokenizer. If the
tokenizer is unavailable or missing required specials, the load raises.
"""
from __future__ import annotations
import logging
import os
from typing import Any
import transformers
from transformers import AutoConfig, PretrainedConfig
__all__ = ["HiggsAudioV3Config"]
logger = logging.getLogger(__name__)
_QWEN3_ROPE_THETA = 1_000_000
_REQUIRED_SPECIALS = ("<|tts|>", "<|text|>", "<|audio|>")
def _build_text_config(raw: Any) -> PretrainedConfig:
if isinstance(raw, PretrainedConfig):
return raw
cfg = dict(raw or {})
model_type = cfg.get("model_type", "qwen3")
if model_type == "qwen3" and cfg.get("rope_theta") is None:
cfg["rope_theta"] = _QWEN3_ROPE_THETA
try:
cfg_cls = transformers.CONFIG_MAPPING[model_type]
except KeyError as exc:
raise ValueError(f"Unknown text backbone model_type {model_type!r}") from exc
return cfg_cls(**cfg)
def _resolve_model_dir(pretrained_model_name_or_path: str) -> str | None:
"""Resolve a model name/path to a local directory containing tokenizer files."""
if os.path.isdir(pretrained_model_name_or_path):
return pretrained_model_name_or_path
try:
from huggingface_hub import try_to_load_from_cache
cached = try_to_load_from_cache(repo_id=pretrained_model_name_or_path, filename="tokenizer_config.json")
if isinstance(cached, str) and os.path.isfile(cached):
return os.path.dirname(cached)
except Exception:
pass
try:
from huggingface_hub.constants import HF_HUB_CACHE
safe = pretrained_model_name_or_path.replace("/", "--")
snapshots = os.path.join(HF_HUB_CACHE, f"models--{safe}", "snapshots")
if os.path.isdir(snapshots):
for rev in os.listdir(snapshots):
candidate = os.path.join(snapshots, rev)
if os.path.isfile(os.path.join(candidate, "tokenizer_config.json")):
return candidate
except Exception:
pass
return None
class HiggsAudioV3Config(PretrainedConfig):
"""Typed config for higgs-audio v3 (HiggsMultimodalQwen3).
``from_pretrained()`` automatically resolves ``<|tts|>``, ``<|text|>``,
``<|audio|>`` and ``eos_token_id`` from the checkpoint tokenizer.
"""
model_type: str = "higgs_multimodal_qwen3"
is_composition = True
def __init__(
self,
audio_encoder_config: dict[str, Any] | None = None,
text_config: dict[str, Any] | PretrainedConfig | None = None,
audio_token_id: int = -100,
mel_per_sample: int = 8,
num_codebooks: int = 8,
codebook_size: int = 1026,
audio_stream_bos_id: int = 1024,
audio_stream_eos_id: int = 1025,
sample_rate: int = 24000,
frame_rate: int = 25,
tts_token_id: int | None = None,
text_token_id: int | None = None,
audio_continuation_id: int | None = None,
enable_flashinfer_api_unwrap: bool = True,
enable_mlp_cudagraph: bool = True,
**kwargs: Any,
) -> None:
# Legacy perf knob removed: Higgs v3 scheduler tokens now come from
# SamplerOutput.sampled_token_ids only.
kwargs.pop("enable_cpu_token_override", None)
self.audio_token_id = audio_token_id
self.mel_per_sample = mel_per_sample
if audio_encoder_config is None:
audio_encoder_config = {
"encoder_type": "discrete",
"num_codebooks": num_codebooks,
"vocab_size": codebook_size,
"tie_word_embeddings": True,
}
self.audio_encoder_config = audio_encoder_config
self.num_codebooks = int(audio_encoder_config.get("num_codebooks", num_codebooks))
if self.num_codebooks <= 0:
raise ValueError(f"num_codebooks must be > 0, got {self.num_codebooks}")
self.codebook_size = int(audio_encoder_config.get("vocab_size", codebook_size))
self.tie_modality_embeddings = bool(audio_encoder_config.get("tie_word_embeddings", True))
self.audio_stream_bos_id = audio_stream_bos_id
self.audio_stream_eos_id = audio_stream_eos_id
self.sample_rate = sample_rate
self.frame_rate = frame_rate
self.text_config = _build_text_config(text_config)
self.audio_hidden_size = int(audio_encoder_config.get("out_dim", self.text_config.hidden_size))
self.tts_token_id = tts_token_id
self.text_token_id = text_token_id
self.audio_continuation_id = audio_continuation_id
self.enable_flashinfer_api_unwrap = bool(enable_flashinfer_api_unwrap)
self.enable_mlp_cudagraph = bool(enable_mlp_cudagraph)
super().__init__(**kwargs)
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs: Any) -> HiggsAudioV3Config:
"""Load config and resolve special token IDs from the checkpoint tokenizer.
Passes the original ``pretrained_model_name_or_path`` (local dir or
HF repo ID) directly to ``AutoTokenizer.from_pretrained()`` so it can
handle cache hits, downloads, and local paths uniformly. Raises if the
tokenizer is missing required specials.
"""
config = super().from_pretrained(pretrained_model_name_or_path, **kwargs)
config.resolve_special_tokens(pretrained_model_name_or_path)
return config
def get_text_config(self, decoder: bool = False) -> PretrainedConfig:
del decoder
return self.text_config
@property
def num_real_codes(self) -> int:
return self.audio_stream_bos_id
@property
def hidden_size(self) -> int:
return self.text_config.hidden_size
def resolve_special_tokens(self, model_path: str) -> None:
"""Resolve <|tts|>, <|text|>, <|audio|> and eos from the HF tokenizer.
Raises ``ValueError`` if any of the 3 required specials is missing
from the tokenizer's added vocabulary.
"""
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
vocab = dict(tokenizer.get_added_vocab())
missing = [t for t in _REQUIRED_SPECIALS if t not in vocab]
if missing:
raise ValueError(
f"Tokenizer at {model_path} is missing required Higgs TTS v3 "
f"special tokens: {missing}. Available added tokens: "
f"{sorted(vocab.keys())[:20]}"
)
self.tts_token_id = vocab["<|tts|>"]
self.text_token_id = vocab["<|text|>"]
self.audio_continuation_id = vocab["<|audio|>"]
if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None:
self.eos_token_id = int(tokenizer.eos_token_id)
AutoConfig.register("higgs_multimodal_qwen3", HiggsAudioV3Config)
@@ -23,11 +23,11 @@ from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_decoder import (
build_higgs_audio_acoustic_decoder,
load_higgs_audio_codec,
)
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.platforms import current_omni_platform
from vllm_omni.transformers_utils.configs.higgs_audio_v3 import (
HiggsAudioV3Config,
)
__all__ = [
"HiggsAudioV3Code2Wav",
@@ -36,10 +36,10 @@ from vllm.model_executor.models.qwen3 import Qwen3Model
from vllm.platforms import current_platform
from vllm.v1.outputs import SamplerOutput
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.transformers_utils.configs.higgs_audio_v3 import (
HiggsAudioV3Config,
)
from vllm_omni.model_executor.models.output_templates import OmniOutput
__all__ = ["HiggsAudioV3TalkerForConditionalGeneration"]
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .hunyuan_image3 import HunyuanImage3ForConditionalGeneration
__all__ = ["HunyuanImage3ForConditionalGeneration"]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,2 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,9 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Lance (ByteDance) — vLLM-Omni model_executor package.
Single-stage topology: a self-contained diffusion stage (``LancePipeline``
defined in ``vllm_omni.diffusion.models.lance``) that owns the Qwen2-MoT LLM,
the Qwen2.5-VL ViT, the Wan2.2 VAE and the tokenizer — directly analogous to
``bagel_single_stage``. The ``PipelineConfig`` lives in :mod:`.pipeline`.
"""
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,17 +1,8 @@
from transformers import AutoTokenizer
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm_omni.tokenizers.mammoth_moda2_tokenizer import MammothUTokenizer
from vllm_omni.transformers_utils.configs.mammoth_moda2 import ( # noqa: F401 registers AutoConfig
Mammothmoda2Config,
Mammothmoda2Qwen2_5_VLConfig,
)
from .mammoth_moda2 import MammothModa2ARForConditionalGeneration
# AutoConfig.register is done inside transformers_utils/configs/mammoth_moda2.py
AutoTokenizer.register(config_class=Mammothmoda2Config, slow_tokenizer_class=MammothUTokenizer)
AutoTokenizer.register(config_class=Mammothmoda2Qwen2_5_VLConfig, slow_tokenizer_class=MammothUTokenizer)
__all__ = [
"MammothModa2ARForConditionalGeneration",
]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,20 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .ming_flash_omni import MingFlashOmniForConditionalGeneration
from .ming_flash_omni_talker import MingFlashOmniTalkerForConditionalGeneration
from .ming_flash_omni_thinker import (
MingFlashOmniThinkerDummyInputsBuilder,
MingFlashOmniThinkerForConditionalGeneration,
MingFlashOmniThinkerMultiModalProcessor,
MingFlashOmniThinkerProcessingInfo,
)
__all__ = [
"MingFlashOmniForConditionalGeneration",
"MingFlashOmniTalkerForConditionalGeneration",
"MingFlashOmniThinkerForConditionalGeneration",
"MingFlashOmniThinkerProcessingInfo",
"MingFlashOmniThinkerMultiModalProcessor",
"MingFlashOmniThinkerDummyInputsBuilder",
]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,13 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .config_ming_tts import MingDenseConfig
from .ming_tts import MingTTSForConditionalGeneration
from .ming_tts_audio_vae import MingAudioVAEModel
from .ming_tts_llm import MingLLMModel
__all__ = [
"MingDenseConfig",
"MingTTSForConditionalGeneration",
"MingLLMModel",
"MingAudioVAEModel",
]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,9 +1,8 @@
from .minicpmo_4_5_omni import MiniCPMO45OmniForConditionalGeneration
from .minicpmo_4_5_omni_llm import MiniCPMO45OmniLLMForConditionalGeneration
from .minicpmo_4_5_omni_tts import MiniCPMO45OmniTTSForConditionalGeneration
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
__all__ = [
"MiniCPMO45OmniForConditionalGeneration",
"MiniCPMO45OmniLLMForConditionalGeneration",
"MiniCPMO45OmniTTSForConditionalGeneration",
]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,13 +1,8 @@
from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_codec import (
MossTTSCodecDecoder,
)
from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_talker import (
MossTTSDelayTalkerForGeneration,
MossTTSRealtimeTalkerForGeneration,
)
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
__all__ = [
"MossTTSDelayTalkerForGeneration",
"MossTTSRealtimeTalkerForGeneration",
"MossTTSCodecDecoder",
]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,5 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .modeling_moss_tts_nano import MossTTSNanoForGeneration
__all__ = ["MossTTSNanoForGeneration"]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -1,3 +1,8 @@
from .qwen3_omni import Qwen3OmniMoeForConditionalGeneration
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
__all__ = ["Qwen3OmniMoeForConditionalGeneration"]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -7,10 +7,13 @@ Stage 1: Talker — text embeddings → RVQ codec codes
Stage 2: Code2Wav — RVQ codes → audio waveform
"""
from transformers import Qwen3OmniMoeConfig
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
pipeline_cfg_resolver,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.qwen3_omni"
@@ -64,3 +67,38 @@ QWEN3_OMNI_PIPELINE = PipelineConfig(
),
),
)
QWEN3_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig(
model_type="qwen3_omni_moe_thinker_only",
model_arch="Qwen3OmniMoeForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="thinker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="text",
owns_tokenizer=True,
requires_multimodal_data=True,
hf_config_name="thinker_config",
engine_output_type="latent",
sampling_constraints={"detokenize": True},
),
),
)
@pipeline_cfg_resolver(config_type=Qwen3OmniMoeConfig)
def resolve_qwen3_omni_pipeline(
hf_config: Qwen3OmniMoeConfig,
) -> PipelineConfig:
"""Select the right pipeline variant based on the HF config, since some variants,
e.g., Qwen3-Omni-30B-A3B-Captioner, are thinker only.
By default, we load the full pipeline, as this is the common case.
"""
# If we have a config and it explicitly disabled audio input, load thinker only
if not hf_config.enable_audio_output:
return QWEN3_OMNI_THINKER_ONLY_PIPELINE
return QWEN3_OMNI_PIPELINE
@@ -106,9 +106,6 @@ class Qwen3OmniMoeTalkerForConditionalGeneration(
self.hidden_projection = Qwen3OmniMoeTalkerResizeMLP(self.config)
self.codec_head = nn.Linear(self.config.text_config.hidden_size, self.config.text_config.vocab_size, bias=False)
self.rope_deltas = None
self.spatial_merge_size = self.config.spatial_merge_size
self.vocab_size = self.config.code_predictor_config.vocab_size
self.num_code_groups = self.config.code_predictor_config.num_code_groups
@@ -1,5 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .voxcpm2_talker import VoxCPM2TalkerForConditionalGeneration
__all__ = ["VoxCPM2TalkerForConditionalGeneration"]
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.