[Chore] Bump Python version and apply formatting changes for v0.12 release (#126)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung authored and GitHub committed 2025-12-16 15:19:40 +08:00
1 parent 31fedcfa2b
commit c091a03d82
61 files changed
+534 -553

No files matched your search

@@ -1,4 +1,4 @@
from typing import NamedTuple, Optional
from typing import NamedTuple
import torch
from vllm.sequence import IntermediateTensors
@@ -8,6 +8,6 @@ class OmniOutput(NamedTuple):
"""Output from the merged Omni model containing both text and audio."""
text_hidden_states: torch.Tensor
multimodal_outputs: Optional[dict] = None
intermediate_tensors: Optional[IntermediateTensors] = None
next_token_id: Optional[torch.Tensor] = None
multimodal_outputs: dict | None = None
intermediate_tensors: IntermediateTensors | None = None
next_token_id: torch.Tensor | None = None
@@ -2,7 +2,6 @@ import glob
import os
from collections.abc import Iterable
from functools import cached_property
from typing import Optional, Union
import numpy as np
import torch
@@ -136,9 +135,9 @@ class Qwen2_5OmniForConditionalGeneration(
def move_submodules_to_devices(
self,
*,
thinker_device: Optional[Union[str, torch.device]] = None,
talker_device: Optional[Union[str, torch.device]] = None,
token2wav_device: Optional[Union[str, torch.device]] = None,
thinker_device: str | torch.device | None = None,
talker_device: str | torch.device | None = None,
token2wav_device: str | torch.device | None = None,
) -> None:
"""Optionally move thinker/talker/token2wav to different devices.
@@ -182,17 +181,17 @@ class Qwen2_5OmniForConditionalGeneration(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
generate_audio: bool = True,
voice_type: str = "Chelsie",
codec: Optional[torch.Tensor] = None,
sampling_metadata: Optional[SamplingMetadata] = None,
logits_index: Optional[int] = None,
codec: torch.Tensor | None = None,
sampling_metadata: SamplingMetadata | None = None,
logits_index: int | None = None,
sampler=None,
additional_information: Optional[dict[str, object]] = None,
additional_information: dict[str, object] | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors, OmniOutput]:
) -> torch.Tensor | IntermediateTensors | OmniOutput:
"""
Workflow:
1) Thinker: multimodal understanding → text hidden states.
@@ -292,9 +291,9 @@ class Qwen2_5OmniForConditionalGeneration(
inputs_embeds = self.talker.get_input_embeddings(input_ids)
# ------- Request-scoped additional information (no cross-request concat) -------
request_ids: Optional[list[str]] = kwargs.get("request_ids") # ordered
request_token_spans: Optional[list[tuple[int, int]]] = kwargs.get("request_token_spans")
addi_by_req: Optional[dict] = kwargs.get("additional_information_by_req_id")
request_ids: list[str] | None = kwargs.get("request_ids") # ordered
request_token_spans: list[tuple[int, int]] | None = kwargs.get("request_token_spans")
addi_by_req: dict | None = kwargs.get("additional_information_by_req_id")
runtime_addi = kwargs.get("runtime_additional_information")
# Normalize runtime_addi into a mapping by request_id for convenience
@@ -796,7 +795,7 @@ class Qwen2_5OmniForConditionalGeneration(
) # for decode
return output_token_ids, processed_output_token_embeds
def compute_logits(self, hidden_states: Union[torch.Tensor, OmniOutput]) -> Optional[torch.Tensor]:
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None:
# Handle OmniOutput type
if isinstance(hidden_states, OmniOutput):
hidden_states = hidden_states.text_hidden_states
@@ -808,7 +807,7 @@ class Qwen2_5OmniForConditionalGeneration(
self,
logits: torch.Tensor,
sampling_metadata: SamplingMetadata,
) -> Optional[SamplerOutput]:
) -> SamplerOutput | None:
# Use thinker model for sampling
return self.model.sample(logits, sampling_metadata)
@@ -891,7 +890,7 @@ class Qwen2_5OmniForConditionalGeneration(
key = os.path.basename(f).split("_")[0].lower()
self._token2wav_ref_mels[key] = torch.as_tensor(np.load(f), device=device)
def _codec_to_audio(self, codec_tokens: torch.Tensor, voice_type: str = "default") -> Optional[torch.Tensor]:
def _codec_to_audio(self, codec_tokens: torch.Tensor, voice_type: str = "default") -> torch.Tensor | None:
if self.token2wav is None:
self._init_token2wav_model()
if self.token2wav is None:
@@ -1,6 +1,5 @@
from collections.abc import Iterable
from functools import cached_property
from typing import Optional, Union
import torch
import torch.nn as nn
@@ -107,7 +106,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
def get_input_embeddings(
self,
input_ids: torch.Tensor,
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
multimodal_embeddings: MultiModalEmbeddings | None = None,
) -> torch.Tensor:
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
if multimodal_embeddings is not None and len(multimodal_embeddings) != 0:
@@ -129,10 +128,10 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
self,
input_ids: torch.Tensor = None,
positions: torch.Tensor = None,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
assert input_ids is not None or inputs_embeds is not None, "input_ids or inputs_embeds must be provided"
# forward_context: ForwardContext = get_forward_context() # unused variable
@@ -158,7 +157,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
logits[..., bos_id] = -1e9
return logits
def compute_logits(self, hidden_states: torch.Tensor) -> Optional[torch.Tensor]:
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
logits = self.language_model.compute_logits(hidden_states)
logits = self.bad_word_processor(logits)
return logits
@@ -167,7 +166,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
self,
logits: torch.Tensor,
sampling_metadata: SamplingMetadata,
) -> Optional[SamplerOutput]:
) -> SamplerOutput | None:
return self.language_model.sample(logits, sampling_metadata)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -1,7 +1,7 @@
from collections import defaultdict
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Callable, Iterable, Mapping, Sequence
from functools import partial
from typing import Annotated, Any, Callable, Literal, Optional, Union
from typing import Annotated, Any, Literal
import torch
import torch.nn as nn
@@ -86,7 +86,7 @@ class Qwen2_5OmniAudioFeatureInputs(TensorSchema):
type: Literal["audio_features"]
input_features: Annotated[
Union[torch.Tensor, list[torch.Tensor]],
torch.Tensor | list[torch.Tensor],
TensorShape("nmb", "tsl"),
]
@@ -155,7 +155,7 @@ def create_qwen2_5_omni_thinker_field_factory(
class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
def _parse_audio_data(
self,
data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any]:
if isinstance(data, dict):
return DictEmbeddingItems(
@@ -185,7 +185,7 @@ class Qwen2_5OmniThinkerProcessingInfo(Qwen2AudioProcessingInfo, Qwen2_5_VLProce
assert isinstance(feature_extractor, WhisperFeatureExtractor)
return feature_extractor
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": None, "image": None, "video": None}
@@ -608,7 +608,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(BaseMultiModalProcessor[Qwen2_5OmniT
def _apply_hf_processor_main(
self,
prompt: Union[str, list[int]],
prompt: str | list[int],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
@@ -676,7 +676,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
else:
return torch.concat(mm_input, dim=dim)
def _parse_and_validate_audio_input(self, **kwargs: object) -> Optional[Qwen2_5OmniAudioFeatureInputs]:
def _parse_and_validate_audio_input(self, **kwargs: object) -> Qwen2_5OmniAudioFeatureInputs | None:
input_audio_features = kwargs.pop("input_audio_features", None)
audio_feature_lengths = kwargs.pop("audio_feature_lengths", None)
feature_attention_mask = kwargs.pop("feature_attention_mask", None)
@@ -699,7 +699,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
def _parse_and_validate_image_input(
self,
**kwargs: dict[str, Any],
) -> Optional[Qwen2_5_VLImageInputs]:
) -> Qwen2_5_VLImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
image_grid_thw = kwargs.pop("image_grid_thw", None)
@@ -731,7 +731,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
def _parse_and_validate_video_input(
self,
**kwargs: dict[str, Any],
) -> Optional[Qwen2_5_VLVideoInputs]:
) -> Qwen2_5_VLVideoInputs | None:
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
video_embeds = kwargs.pop("video_embeds", None)
video_grid_thw = kwargs.pop("video_grid_thw", None)
@@ -841,7 +841,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|vision_start|><|IMAGE|><|vision_end|>"
if modality.startswith("video"):
@@ -940,7 +940,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
def get_input_embeddings(
self,
input_ids: torch.Tensor,
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
multimodal_embeddings: MultiModalEmbeddings | None = None,
) -> torch.Tensor:
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
if multimodal_embeddings is not None and len(multimodal_embeddings) != 0:
@@ -958,7 +958,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
)
return inputs_embeds
def get_multimodal_embeddings_v0(self, **kwargs: object) -> Optional[NestedTensors]:
def get_multimodal_embeddings_v0(self, **kwargs: object) -> NestedTensors | None:
audio_input = self._parse_and_validate_audio_input(**kwargs)
image_input = self._parse_and_validate_image_input(**kwargs)
video_input = self._parse_and_validate_video_input(**kwargs)
@@ -982,7 +982,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
def get_input_embeddings_v0(
self,
input_ids: torch.Tensor,
multimodal_embeddings: Optional[NestedTensors] = None,
multimodal_embeddings: NestedTensors | None = None,
) -> torch.Tensor:
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
@@ -1002,10 +1002,10 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
@@ -1031,7 +1031,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
)
return text_inputs_embeds, hidden_states.unsqueeze(0) # (1, S, D)
def compute_logits(self, hidden_states: torch.Tensor) -> Optional[torch.Tensor]:
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -4,7 +4,6 @@
import math
from collections.abc import Iterable
from typing import Optional, Union
import numpy as np
import torch
@@ -395,9 +394,9 @@ class DiTInputEmbedding(nn.Module):
speaker_embedding: torch.Tensor,
condition_vector: torch.Tensor,
code_embed: torch.Tensor,
drop_audio_cond: Optional[bool] = False,
code_embed_uncond: Optional[bool] = None,
apply_cfg: Optional[bool] = True,
drop_audio_cond: bool | None = False,
code_embed_uncond: bool | None = None,
apply_cfg: bool | None = True,
):
if apply_cfg:
hidden_states = torch.cat([hidden_states, hidden_states], dim=0)
@@ -1324,7 +1323,7 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
y0: torch.Tensor,
num_steps: int = 10,
guidance_scale: float = 0.5,
sway_coefficient: Optional[float] = -1.0,
sway_coefficient: float | None = -1.0,
) -> torch.Tensor:
"""
Block-wise ODE sampling starting from provided initial state y0.
@@ -1521,7 +1520,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
steps: int,
prev_generated: torch.Tensor,
finished: bool = False,
) -> tuple[Optional[torch.Tensor], torch.Tensor]:
) -> tuple[torch.Tensor | None, torch.Tensor]:
"""Streaming per small chunk: returns (mel_or_None, audio_slice)."""
start_index = max(i * self.chunk_size - self.past_cache_size, 0)
end_index = min(
@@ -1560,9 +1559,9 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
y_all: torch.Tensor,
i: int,
steps: int,
prev_generated: Union[torch.Tensor, list[torch.Tensor]],
prev_generated: torch.Tensor | list[torch.Tensor],
finished: bool = False,
) -> tuple[Union[torch.Tensor, list[torch.Tensor]], torch.Tensor]:
) -> tuple[torch.Tensor | list[torch.Tensor], torch.Tensor]:
"""High-level chunk API aligning to qwen2_code2wav_dit signature."""
if not isinstance(prev_generated, torch.Tensor):
prev_generated = prev_generated[0] if len(prev_generated) > 0 else None
@@ -1585,7 +1584,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
start_index: int,
end_index: int,
finished: bool,
prev_generated: Optional[torch.Tensor],
prev_generated: torch.Tensor | None,
generated: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
@@ -1671,7 +1670,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
num_steps: int = 10,
guidance_scale: float = 0.5,
sway_coefficient: float = -1.0,
intermediate_tensors: Optional[IntermediateTensors] = None,
intermediate_tensors: IntermediateTensors | None = None,
**kwargs,
) -> torch.Tensor:
# Delegate to HF token2wav model
@@ -1685,7 +1684,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
**kwargs,
)
def compute_logits(self, hidden_states: torch.Tensor) -> Optional[torch.Tensor]:
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
# Token2Wav outputs waveform; logits are not applicable
return hidden_states
@@ -1693,7 +1692,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
self,
logits: torch.Tensor,
sampling_metadata: SamplingMetadata,
) -> Optional[SamplerOutput]:
) -> SamplerOutput | None:
return None
def load_weights_without_buffers(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -1782,7 +1781,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
)
@torch.inference_mode()
def process_chunk_bigvgan_batch(self, mel_batch: torch.Tensor) -> Optional[torch.Tensor]:
def process_chunk_bigvgan_batch(self, mel_batch: torch.Tensor) -> torch.Tensor | None:
# BigVGAN is not part of this wrapper; return None for parity.
return None
@@ -1797,7 +1796,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
steps: int,
prev_generated: torch.Tensor,
finished: bool = False,
) -> tuple[Optional[torch.Tensor], torch.Tensor]:
) -> tuple[torch.Tensor | None, torch.Tensor]:
mel = self.token2wav(
code=codec_all,
conditioning=conditioning,
@@ -1815,9 +1814,9 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
y_all: torch.Tensor,
i: int,
steps: int,
prev_generated: Union[torch.Tensor, list[torch.Tensor]],
prev_generated: torch.Tensor | list[torch.Tensor],
finished: bool = False,
) -> tuple[Union[torch.Tensor, list[torch.Tensor]], torch.Tensor]:
) -> tuple[torch.Tensor | list[torch.Tensor], torch.Tensor]:
_mel, out = self.process_little_chunk(
conditioning=conditioning,
reference_mel=reference_mel,
@@ -1,5 +1,4 @@
from collections.abc import Iterable
from typing import Optional, Union
import torch
from torch import nn
@@ -43,7 +42,7 @@ class Qwen2MLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -80,10 +79,10 @@ class Qwen2Attention(nn.Module):
num_kv_heads: int,
max_position: int = 4096 * 32,
rope_theta: float = 10000,
head_dim: Optional[int] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
rope_scaling: Optional[tuple] = None,
head_dim: int | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
rope_scaling: tuple | None = None,
prefix: str = "",
attn_type: str = AttentionType.DECODER,
) -> None:
@@ -161,8 +160,8 @@ class Qwen2DecoderLayer(nn.Module):
def __init__(
self,
config: Qwen2Config,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -207,7 +206,7 @@ class Qwen2DecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
@@ -302,9 +301,9 @@ class Qwen2Model(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -428,16 +427,16 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
@@ -445,7 +444,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
logits: torch.Tensor,
sampling_metadata: SamplingMetadata,
) -> Optional[SamplerOutput]:
) -> SamplerOutput | None:
next_tokens = self.sampler(logits, sampling_metadata)
return next_tokens
@@ -503,7 +502,7 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
intermediate_tensors: IntermediateTensors | None = None,
) -> torch.Tensor:
return self.model(input_ids, positions, intermediate_tensors)
@@ -511,7 +510,7 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> Optional[PoolerOutput]:
) -> PoolerOutput | None:
return self._pooler(hidden_states, pooling_metadata)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
@@ -6,7 +6,6 @@
from collections import defaultdict
from collections.abc import Iterable
from functools import cached_property
from typing import Optional, Union
import torch
import torch.nn as nn
@@ -198,10 +197,10 @@ class Qwen3OmniMoeForConditionalGeneration(
def move_submodules_to_devices(
self,
*,
thinker_device: Optional[Union[str, torch.device]] = None,
talker_device: Optional[Union[str, torch.device]] = None,
code_predictor_device: Optional[Union[str, torch.device]] = None,
code2wav_device: Optional[Union[str, torch.device]] = None,
thinker_device: str | torch.device | None = None,
talker_device: str | torch.device | None = None,
code_predictor_device: str | torch.device | None = None,
code2wav_device: str | torch.device | None = None,
) -> None:
"""
Optionally move thinker/talker/code2wav to different devices.
@@ -257,16 +256,16 @@ class Qwen3OmniMoeForConditionalGeneration(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
generate_audio: bool = True,
voice_type: str = "ethan",
codec: Optional[torch.Tensor] = None,
sampling_metadata: Optional[SamplingMetadata] = None,
logits_index: Optional[int] = None,
additional_information: Optional[dict[str, object]] = None,
codec: torch.Tensor | None = None,
sampling_metadata: SamplingMetadata | None = None,
logits_index: int | None = None,
additional_information: dict[str, object] | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors, OmniOutput]:
) -> torch.Tensor | IntermediateTensors | OmniOutput:
"""
Unified forward pass for all model stages.
@@ -383,9 +382,9 @@ class Qwen3OmniMoeForConditionalGeneration(
inputs_embeds = self.talker.get_input_embeddings(input_ids)
# ------- Request-scoped additional information (no cross-request concat) -------
request_ids: Optional[list[str]] = kwargs.get("request_ids") # ordered
request_token_spans: Optional[list[tuple[int, int]]] = kwargs.get("request_token_spans")
addi_by_req: Optional[dict] = kwargs.get("additional_information_by_req_id")
request_ids: list[str] | None = kwargs.get("request_ids") # ordered
request_token_spans: list[tuple[int, int]] | None = kwargs.get("request_token_spans")
addi_by_req: dict | None = kwargs.get("additional_information_by_req_id")
runtime_addi = kwargs.get("runtime_additional_information")
# Normalize runtime_addi into a mapping by request_id for convenience
@@ -808,14 +807,14 @@ class Qwen3OmniMoeForConditionalGeneration(
self,
thinker_embed: torch.Tensor,
thinker_hidden: torch.Tensor,
multimodal_mask: Optional[torch.Tensor],
multimodal_mask: torch.Tensor | None,
input_ids: torch.Tensor,
thinker_result_ids: torch.Tensor,
speaker_id,
tts_bos_thinker: Optional[torch.Tensor] = None,
tts_eos_thinker: Optional[torch.Tensor] = None,
tts_pad_thinker: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
tts_bos_thinker: torch.Tensor | None = None,
tts_eos_thinker: torch.Tensor | None = None,
tts_pad_thinker: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
"""
Project thinker outputs to talker inputs during prefill stage.
@@ -845,7 +844,7 @@ class Qwen3OmniMoeForConditionalGeneration(
return x[-1]
return x.view(1, 1, -1)
def _proj_from_thinker(x_opt: Optional[torch.Tensor]) -> torch.Tensor:
def _proj_from_thinker(x_opt: torch.Tensor | None) -> torch.Tensor:
if isinstance(x_opt, torch.Tensor) and x_opt.numel() > 0:
xin = _ensure_1x1(x_opt).to(talker_dev)
else:
@@ -863,7 +862,7 @@ class Qwen3OmniMoeForConditionalGeneration(
talker_input_embeds = [] # [1 t d]
talker_input_ids = []
trailing_text_hidden_all: Optional[torch.Tensor] = None
trailing_text_hidden_all: torch.Tensor | None = None
# For every chatml parts
for i in range(len(im_start_indexes) - 1):
im_start_index = im_start_indexes[i].item()
@@ -986,8 +985,8 @@ class Qwen3OmniMoeForConditionalGeneration(
def _talker_to_code_predictor(
self,
talker_hidden_states: Optional[torch.Tensor],
layer0_token_ids: Optional[torch.Tensor],
talker_hidden_states: torch.Tensor | None,
layer0_token_ids: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Project talker outputs to code predictor inputs.
@@ -1025,9 +1024,9 @@ class Qwen3OmniMoeForConditionalGeneration(
def compute_logits(
self,
hidden_states: Union[torch.Tensor, OmniOutput],
hidden_states: torch.Tensor | OmniOutput,
sampling_metadata: SamplingMetadata = None,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""Compute logits from hidden states."""
# Handle OmniOutput type
from vllm.v1.sample.logits_processor import LogitsProcessors
@@ -1083,7 +1082,7 @@ class Qwen3OmniMoeForConditionalGeneration(
self,
logits: torch.Tensor,
sampling_metadata: SamplingMetadata,
) -> Optional[SamplerOutput]:
) -> SamplerOutput | None:
"""Sample from logits."""
return self.model.sample(logits, sampling_metadata)
@@ -7,7 +7,7 @@ autoregressively, predicting layers 1 to N based on layer-0 codes from the talke
"""
from collections import namedtuple
from typing import Any, Optional
from typing import Any
import torch
import torch.nn as nn
@@ -136,10 +136,10 @@ class Qwen3OmniCodePredictorAttention(nn.Module):
causal_mask: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
past_key_values: Optional[Cache] = None,
cache_position: Optional[torch.LongTensor] = None,
past_key_values: Cache | None = None,
cache_position: torch.LongTensor | None = None,
use_cache: bool = False,
position_ids: Optional[torch.LongTensor] = None,
position_ids: torch.LongTensor | None = None,
) -> torch.Tensor:
bsz, seq_len, _ = hidden_states.shape
@@ -253,8 +253,8 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
prefix: str,
model_config: ModelConfig,
layer_idx: int,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.layer_idx = layer_idx
@@ -284,10 +284,10 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
causal_mask: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
past_key_values: Optional[Cache] = None,
cache_position: Optional[torch.LongTensor] = None,
past_key_values: Cache | None = None,
cache_position: torch.LongTensor | None = None,
use_cache: bool = False,
position_ids: Optional[torch.LongTensor] = None,
position_ids: torch.LongTensor | None = None,
) -> torch.Tensor:
# Self-attention with residual
residual = hidden_states
@@ -310,7 +310,7 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_index: int = 0,
) -> torch.Tensor:
assert inputs_embeds is not None, "inputs_embeds required for MTP"
@@ -396,11 +396,11 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module):
def forward(
self,
inputs_embeds: torch.Tensor,
attention_mask: Optional[torch.Tensor] = None,
position_ids: Optional[torch.LongTensor] = None,
past_key_values: Optional[Any] = None,
use_cache: Optional[bool] = False,
cache_position: Optional[torch.LongTensor] = None,
attention_mask: torch.Tensor | None = None,
position_ids: torch.LongTensor | None = None,
past_key_values: Any | None = None,
use_cache: bool | None = False,
cache_position: torch.LongTensor | None = None,
**kwargs: Any,
) -> Any:
"""