[Chore] Bump Python version and apply formatting changes for v0.12 release (#126)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
1 parent
31fedcfa2b
commit
c091a03d82
61 files changed
+534
-553
No files matched your search
@@ -1,4 +1,4 @@
|
||||
from typing import NamedTuple, Optional
|
||||
from typing import NamedTuple
|
||||
|
||||
import torch
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -8,6 +8,6 @@ class OmniOutput(NamedTuple):
|
||||
"""Output from the merged Omni model containing both text and audio."""
|
||||
|
||||
text_hidden_states: torch.Tensor
|
||||
multimodal_outputs: Optional[dict] = None
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None
|
||||
next_token_id: Optional[torch.Tensor] = None
|
||||
multimodal_outputs: dict | None = None
|
||||
intermediate_tensors: IntermediateTensors | None = None
|
||||
next_token_id: torch.Tensor | None = None
|
||||
@@ -2,7 +2,6 @@ import glob
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from functools import cached_property
|
||||
from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -136,9 +135,9 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
def move_submodules_to_devices(
|
||||
self,
|
||||
*,
|
||||
thinker_device: Optional[Union[str, torch.device]] = None,
|
||||
talker_device: Optional[Union[str, torch.device]] = None,
|
||||
token2wav_device: Optional[Union[str, torch.device]] = None,
|
||||
thinker_device: str | torch.device | None = None,
|
||||
talker_device: str | torch.device | None = None,
|
||||
token2wav_device: str | torch.device | None = None,
|
||||
) -> None:
|
||||
"""Optionally move thinker/talker/token2wav to different devices.
|
||||
|
||||
@@ -182,17 +181,17 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
generate_audio: bool = True,
|
||||
voice_type: str = "Chelsie",
|
||||
codec: Optional[torch.Tensor] = None,
|
||||
sampling_metadata: Optional[SamplingMetadata] = None,
|
||||
logits_index: Optional[int] = None,
|
||||
codec: torch.Tensor | None = None,
|
||||
sampling_metadata: SamplingMetadata | None = None,
|
||||
logits_index: int | None = None,
|
||||
sampler=None,
|
||||
additional_information: Optional[dict[str, object]] = None,
|
||||
additional_information: dict[str, object] | None = None,
|
||||
**kwargs: object,
|
||||
) -> Union[torch.Tensor, IntermediateTensors, OmniOutput]:
|
||||
) -> torch.Tensor | IntermediateTensors | OmniOutput:
|
||||
"""
|
||||
Workflow:
|
||||
1) Thinker: multimodal understanding → text hidden states.
|
||||
@@ -292,9 +291,9 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
inputs_embeds = self.talker.get_input_embeddings(input_ids)
|
||||
|
||||
# ------- Request-scoped additional information (no cross-request concat) -------
|
||||
request_ids: Optional[list[str]] = kwargs.get("request_ids") # ordered
|
||||
request_token_spans: Optional[list[tuple[int, int]]] = kwargs.get("request_token_spans")
|
||||
addi_by_req: Optional[dict] = kwargs.get("additional_information_by_req_id")
|
||||
request_ids: list[str] | None = kwargs.get("request_ids") # ordered
|
||||
request_token_spans: list[tuple[int, int]] | None = kwargs.get("request_token_spans")
|
||||
addi_by_req: dict | None = kwargs.get("additional_information_by_req_id")
|
||||
runtime_addi = kwargs.get("runtime_additional_information")
|
||||
|
||||
# Normalize runtime_addi into a mapping by request_id for convenience
|
||||
@@ -796,7 +795,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
) # for decode
|
||||
return output_token_ids, processed_output_token_embeds
|
||||
|
||||
def compute_logits(self, hidden_states: Union[torch.Tensor, OmniOutput]) -> Optional[torch.Tensor]:
|
||||
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None:
|
||||
# Handle OmniOutput type
|
||||
if isinstance(hidden_states, OmniOutput):
|
||||
hidden_states = hidden_states.text_hidden_states
|
||||
@@ -808,7 +807,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> Optional[SamplerOutput]:
|
||||
) -> SamplerOutput | None:
|
||||
# Use thinker model for sampling
|
||||
return self.model.sample(logits, sampling_metadata)
|
||||
|
||||
@@ -891,7 +890,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
key = os.path.basename(f).split("_")[0].lower()
|
||||
self._token2wav_ref_mels[key] = torch.as_tensor(np.load(f), device=device)
|
||||
|
||||
def _codec_to_audio(self, codec_tokens: torch.Tensor, voice_type: str = "default") -> Optional[torch.Tensor]:
|
||||
def _codec_to_audio(self, codec_tokens: torch.Tensor, voice_type: str = "default") -> torch.Tensor | None:
|
||||
if self.token2wav is None:
|
||||
self._init_token2wav_model()
|
||||
if self.token2wav is None:
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from collections.abc import Iterable
|
||||
from functools import cached_property
|
||||
from typing import Optional, Union
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -107,7 +106,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
|
||||
def get_input_embeddings(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
|
||||
if multimodal_embeddings is not None and len(multimodal_embeddings) != 0:
|
||||
@@ -129,10 +128,10 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
|
||||
self,
|
||||
input_ids: torch.Tensor = None,
|
||||
positions: torch.Tensor = None,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
) -> Union[torch.Tensor, IntermediateTensors]:
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
assert input_ids is not None or inputs_embeds is not None, "input_ids or inputs_embeds must be provided"
|
||||
# forward_context: ForwardContext = get_forward_context() # unused variable
|
||||
|
||||
@@ -158,7 +157,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
|
||||
logits[..., bos_id] = -1e9
|
||||
return logits
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> Optional[torch.Tensor]:
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
logits = self.language_model.compute_logits(hidden_states)
|
||||
logits = self.bad_word_processor(logits)
|
||||
return logits
|
||||
@@ -167,7 +166,7 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> Optional[SamplerOutput]:
|
||||
) -> SamplerOutput | None:
|
||||
return self.language_model.sample(logits, sampling_metadata)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from functools import partial
|
||||
from typing import Annotated, Any, Callable, Literal, Optional, Union
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -86,7 +86,7 @@ class Qwen2_5OmniAudioFeatureInputs(TensorSchema):
|
||||
|
||||
type: Literal["audio_features"]
|
||||
input_features: Annotated[
|
||||
Union[torch.Tensor, list[torch.Tensor]],
|
||||
torch.Tensor | list[torch.Tensor],
|
||||
TensorShape("nmb", "tsl"),
|
||||
]
|
||||
|
||||
@@ -155,7 +155,7 @@ def create_qwen2_5_omni_thinker_field_factory(
|
||||
class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
|
||||
def _parse_audio_data(
|
||||
self,
|
||||
data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any]:
|
||||
if isinstance(data, dict):
|
||||
return DictEmbeddingItems(
|
||||
@@ -185,7 +185,7 @@ class Qwen2_5OmniThinkerProcessingInfo(Qwen2AudioProcessingInfo, Qwen2_5_VLProce
|
||||
assert isinstance(feature_extractor, WhisperFeatureExtractor)
|
||||
return feature_extractor
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"audio": None, "image": None, "video": None}
|
||||
|
||||
|
||||
@@ -608,7 +608,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(BaseMultiModalProcessor[Qwen2_5OmniT
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: Union[str, list[int]],
|
||||
prompt: str | list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
@@ -676,7 +676,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
else:
|
||||
return torch.concat(mm_input, dim=dim)
|
||||
|
||||
def _parse_and_validate_audio_input(self, **kwargs: object) -> Optional[Qwen2_5OmniAudioFeatureInputs]:
|
||||
def _parse_and_validate_audio_input(self, **kwargs: object) -> Qwen2_5OmniAudioFeatureInputs | None:
|
||||
input_audio_features = kwargs.pop("input_audio_features", None)
|
||||
audio_feature_lengths = kwargs.pop("audio_feature_lengths", None)
|
||||
feature_attention_mask = kwargs.pop("feature_attention_mask", None)
|
||||
@@ -699,7 +699,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
def _parse_and_validate_image_input(
|
||||
self,
|
||||
**kwargs: dict[str, Any],
|
||||
) -> Optional[Qwen2_5_VLImageInputs]:
|
||||
) -> Qwen2_5_VLImageInputs | None:
|
||||
pixel_values = kwargs.pop("pixel_values", None)
|
||||
image_embeds = kwargs.pop("image_embeds", None)
|
||||
image_grid_thw = kwargs.pop("image_grid_thw", None)
|
||||
@@ -731,7 +731,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
def _parse_and_validate_video_input(
|
||||
self,
|
||||
**kwargs: dict[str, Any],
|
||||
) -> Optional[Qwen2_5_VLVideoInputs]:
|
||||
) -> Qwen2_5_VLVideoInputs | None:
|
||||
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
|
||||
video_embeds = kwargs.pop("video_embeds", None)
|
||||
video_grid_thw = kwargs.pop("video_grid_thw", None)
|
||||
@@ -841,7 +841,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality.startswith("image"):
|
||||
return "<|vision_start|><|IMAGE|><|vision_end|>"
|
||||
if modality.startswith("video"):
|
||||
@@ -940,7 +940,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
def get_input_embeddings(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
|
||||
if multimodal_embeddings is not None and len(multimodal_embeddings) != 0:
|
||||
@@ -958,7 +958,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
)
|
||||
return inputs_embeds
|
||||
|
||||
def get_multimodal_embeddings_v0(self, **kwargs: object) -> Optional[NestedTensors]:
|
||||
def get_multimodal_embeddings_v0(self, **kwargs: object) -> NestedTensors | None:
|
||||
audio_input = self._parse_and_validate_audio_input(**kwargs)
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
video_input = self._parse_and_validate_video_input(**kwargs)
|
||||
@@ -982,7 +982,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
def get_input_embeddings_v0(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: Optional[NestedTensors] = None,
|
||||
multimodal_embeddings: NestedTensors | None = None,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
@@ -1002,10 +1002,10 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
) -> Union[torch.Tensor, IntermediateTensors]:
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
|
||||
@@ -1031,7 +1031,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
)
|
||||
return text_inputs_embeds, hidden_states.unsqueeze(0) # (1, S, D)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> Optional[torch.Tensor]:
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable
|
||||
from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -395,9 +394,9 @@ class DiTInputEmbedding(nn.Module):
|
||||
speaker_embedding: torch.Tensor,
|
||||
condition_vector: torch.Tensor,
|
||||
code_embed: torch.Tensor,
|
||||
drop_audio_cond: Optional[bool] = False,
|
||||
code_embed_uncond: Optional[bool] = None,
|
||||
apply_cfg: Optional[bool] = True,
|
||||
drop_audio_cond: bool | None = False,
|
||||
code_embed_uncond: bool | None = None,
|
||||
apply_cfg: bool | None = True,
|
||||
):
|
||||
if apply_cfg:
|
||||
hidden_states = torch.cat([hidden_states, hidden_states], dim=0)
|
||||
@@ -1324,7 +1323,7 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel):
|
||||
y0: torch.Tensor,
|
||||
num_steps: int = 10,
|
||||
guidance_scale: float = 0.5,
|
||||
sway_coefficient: Optional[float] = -1.0,
|
||||
sway_coefficient: float | None = -1.0,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Block-wise ODE sampling starting from provided initial state y0.
|
||||
@@ -1521,7 +1520,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
|
||||
steps: int,
|
||||
prev_generated: torch.Tensor,
|
||||
finished: bool = False,
|
||||
) -> tuple[Optional[torch.Tensor], torch.Tensor]:
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""Streaming per small chunk: returns (mel_or_None, audio_slice)."""
|
||||
start_index = max(i * self.chunk_size - self.past_cache_size, 0)
|
||||
end_index = min(
|
||||
@@ -1560,9 +1559,9 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
|
||||
y_all: torch.Tensor,
|
||||
i: int,
|
||||
steps: int,
|
||||
prev_generated: Union[torch.Tensor, list[torch.Tensor]],
|
||||
prev_generated: torch.Tensor | list[torch.Tensor],
|
||||
finished: bool = False,
|
||||
) -> tuple[Union[torch.Tensor, list[torch.Tensor]], torch.Tensor]:
|
||||
) -> tuple[torch.Tensor | list[torch.Tensor], torch.Tensor]:
|
||||
"""High-level chunk API aligning to qwen2_code2wav_dit signature."""
|
||||
if not isinstance(prev_generated, torch.Tensor):
|
||||
prev_generated = prev_generated[0] if len(prev_generated) > 0 else None
|
||||
@@ -1585,7 +1584,7 @@ class Qwen2_5OmniToken2WavModel(Qwen2_5OmniPreTrainedModel):
|
||||
start_index: int,
|
||||
end_index: int,
|
||||
finished: bool,
|
||||
prev_generated: Optional[torch.Tensor],
|
||||
prev_generated: torch.Tensor | None,
|
||||
generated: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
@@ -1671,7 +1670,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
num_steps: int = 10,
|
||||
guidance_scale: float = 0.5,
|
||||
sway_coefficient: float = -1.0,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
# Delegate to HF token2wav model
|
||||
@@ -1685,7 +1684,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> Optional[torch.Tensor]:
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
# Token2Wav outputs waveform; logits are not applicable
|
||||
return hidden_states
|
||||
|
||||
@@ -1693,7 +1692,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> Optional[SamplerOutput]:
|
||||
) -> SamplerOutput | None:
|
||||
return None
|
||||
|
||||
def load_weights_without_buffers(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
@@ -1782,7 +1781,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def process_chunk_bigvgan_batch(self, mel_batch: torch.Tensor) -> Optional[torch.Tensor]:
|
||||
def process_chunk_bigvgan_batch(self, mel_batch: torch.Tensor) -> torch.Tensor | None:
|
||||
# BigVGAN is not part of this wrapper; return None for parity.
|
||||
return None
|
||||
|
||||
@@ -1797,7 +1796,7 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
steps: int,
|
||||
prev_generated: torch.Tensor,
|
||||
finished: bool = False,
|
||||
) -> tuple[Optional[torch.Tensor], torch.Tensor]:
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
mel = self.token2wav(
|
||||
code=codec_all,
|
||||
conditioning=conditioning,
|
||||
@@ -1815,9 +1814,9 @@ class Qwen2_5OmniToken2WavForConditionalGenerationVLLM(nn.Module, SupportsPP):
|
||||
y_all: torch.Tensor,
|
||||
i: int,
|
||||
steps: int,
|
||||
prev_generated: Union[torch.Tensor, list[torch.Tensor]],
|
||||
prev_generated: torch.Tensor | list[torch.Tensor],
|
||||
finished: bool = False,
|
||||
) -> tuple[Union[torch.Tensor, list[torch.Tensor]], torch.Tensor]:
|
||||
) -> tuple[torch.Tensor | list[torch.Tensor], torch.Tensor]:
|
||||
_mel, out = self.process_little_chunk(
|
||||
conditioning=conditioning,
|
||||
reference_mel=reference_mel,
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
from collections.abc import Iterable
|
||||
from typing import Optional, Union
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -43,7 +42,7 @@ class Qwen2MLP(nn.Module):
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
hidden_act: str,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -80,10 +79,10 @@ class Qwen2Attention(nn.Module):
|
||||
num_kv_heads: int,
|
||||
max_position: int = 4096 * 32,
|
||||
rope_theta: float = 10000,
|
||||
head_dim: Optional[int] = None,
|
||||
cache_config: Optional[CacheConfig] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
rope_scaling: Optional[tuple] = None,
|
||||
head_dim: int | None = None,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
rope_scaling: tuple | None = None,
|
||||
prefix: str = "",
|
||||
attn_type: str = AttentionType.DECODER,
|
||||
) -> None:
|
||||
@@ -161,8 +160,8 @@ class Qwen2DecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen2Config,
|
||||
cache_config: Optional[CacheConfig] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -207,7 +206,7 @@ class Qwen2DecoderLayer(nn.Module):
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
residual: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Self Attention
|
||||
if residual is None:
|
||||
@@ -302,9 +301,9 @@ class Qwen2Model(nn.Module):
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
) -> Union[torch.Tensor, IntermediateTensors]:
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
@@ -428,16 +427,16 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
) -> Union[torch.Tensor, IntermediateTensors]:
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
hidden_states = self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> Optional[torch.Tensor]:
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
@@ -445,7 +444,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> Optional[SamplerOutput]:
|
||||
) -> SamplerOutput | None:
|
||||
next_tokens = self.sampler(logits, sampling_metadata)
|
||||
return next_tokens
|
||||
|
||||
@@ -503,7 +502,7 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.model(input_ids, positions, intermediate_tensors)
|
||||
|
||||
@@ -511,7 +510,7 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> Optional[PoolerOutput]:
|
||||
) -> PoolerOutput | None:
|
||||
return self._pooler(hidden_states, pooling_metadata)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from functools import cached_property
|
||||
from typing import Optional, Union
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -198,10 +197,10 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
def move_submodules_to_devices(
|
||||
self,
|
||||
*,
|
||||
thinker_device: Optional[Union[str, torch.device]] = None,
|
||||
talker_device: Optional[Union[str, torch.device]] = None,
|
||||
code_predictor_device: Optional[Union[str, torch.device]] = None,
|
||||
code2wav_device: Optional[Union[str, torch.device]] = None,
|
||||
thinker_device: str | torch.device | None = None,
|
||||
talker_device: str | torch.device | None = None,
|
||||
code_predictor_device: str | torch.device | None = None,
|
||||
code2wav_device: str | torch.device | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Optionally move thinker/talker/code2wav to different devices.
|
||||
@@ -257,16 +256,16 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
generate_audio: bool = True,
|
||||
voice_type: str = "ethan",
|
||||
codec: Optional[torch.Tensor] = None,
|
||||
sampling_metadata: Optional[SamplingMetadata] = None,
|
||||
logits_index: Optional[int] = None,
|
||||
additional_information: Optional[dict[str, object]] = None,
|
||||
codec: torch.Tensor | None = None,
|
||||
sampling_metadata: SamplingMetadata | None = None,
|
||||
logits_index: int | None = None,
|
||||
additional_information: dict[str, object] | None = None,
|
||||
**kwargs: object,
|
||||
) -> Union[torch.Tensor, IntermediateTensors, OmniOutput]:
|
||||
) -> torch.Tensor | IntermediateTensors | OmniOutput:
|
||||
"""
|
||||
Unified forward pass for all model stages.
|
||||
|
||||
@@ -383,9 +382,9 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
inputs_embeds = self.talker.get_input_embeddings(input_ids)
|
||||
|
||||
# ------- Request-scoped additional information (no cross-request concat) -------
|
||||
request_ids: Optional[list[str]] = kwargs.get("request_ids") # ordered
|
||||
request_token_spans: Optional[list[tuple[int, int]]] = kwargs.get("request_token_spans")
|
||||
addi_by_req: Optional[dict] = kwargs.get("additional_information_by_req_id")
|
||||
request_ids: list[str] | None = kwargs.get("request_ids") # ordered
|
||||
request_token_spans: list[tuple[int, int]] | None = kwargs.get("request_token_spans")
|
||||
addi_by_req: dict | None = kwargs.get("additional_information_by_req_id")
|
||||
runtime_addi = kwargs.get("runtime_additional_information")
|
||||
|
||||
# Normalize runtime_addi into a mapping by request_id for convenience
|
||||
@@ -808,14 +807,14 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
self,
|
||||
thinker_embed: torch.Tensor,
|
||||
thinker_hidden: torch.Tensor,
|
||||
multimodal_mask: Optional[torch.Tensor],
|
||||
multimodal_mask: torch.Tensor | None,
|
||||
input_ids: torch.Tensor,
|
||||
thinker_result_ids: torch.Tensor,
|
||||
speaker_id,
|
||||
tts_bos_thinker: Optional[torch.Tensor] = None,
|
||||
tts_eos_thinker: Optional[torch.Tensor] = None,
|
||||
tts_pad_thinker: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
|
||||
tts_bos_thinker: torch.Tensor | None = None,
|
||||
tts_eos_thinker: torch.Tensor | None = None,
|
||||
tts_pad_thinker: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""
|
||||
Project thinker outputs to talker inputs during prefill stage.
|
||||
|
||||
@@ -845,7 +844,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
return x[-1]
|
||||
return x.view(1, 1, -1)
|
||||
|
||||
def _proj_from_thinker(x_opt: Optional[torch.Tensor]) -> torch.Tensor:
|
||||
def _proj_from_thinker(x_opt: torch.Tensor | None) -> torch.Tensor:
|
||||
if isinstance(x_opt, torch.Tensor) and x_opt.numel() > 0:
|
||||
xin = _ensure_1x1(x_opt).to(talker_dev)
|
||||
else:
|
||||
@@ -863,7 +862,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
|
||||
talker_input_embeds = [] # [1 t d]
|
||||
talker_input_ids = []
|
||||
trailing_text_hidden_all: Optional[torch.Tensor] = None
|
||||
trailing_text_hidden_all: torch.Tensor | None = None
|
||||
# For every chatml parts
|
||||
for i in range(len(im_start_indexes) - 1):
|
||||
im_start_index = im_start_indexes[i].item()
|
||||
@@ -986,8 +985,8 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
|
||||
def _talker_to_code_predictor(
|
||||
self,
|
||||
talker_hidden_states: Optional[torch.Tensor],
|
||||
layer0_token_ids: Optional[torch.Tensor],
|
||||
talker_hidden_states: torch.Tensor | None,
|
||||
layer0_token_ids: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Project talker outputs to code predictor inputs.
|
||||
@@ -1025,9 +1024,9 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: Union[torch.Tensor, OmniOutput],
|
||||
hidden_states: torch.Tensor | OmniOutput,
|
||||
sampling_metadata: SamplingMetadata = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
) -> torch.Tensor | None:
|
||||
"""Compute logits from hidden states."""
|
||||
# Handle OmniOutput type
|
||||
from vllm.v1.sample.logits_processor import LogitsProcessors
|
||||
@@ -1083,7 +1082,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> Optional[SamplerOutput]:
|
||||
) -> SamplerOutput | None:
|
||||
"""Sample from logits."""
|
||||
return self.model.sample(logits, sampling_metadata)
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ autoregressively, predicting layers 1 to N based on layer-0 codes from the talke
|
||||
"""
|
||||
|
||||
from collections import namedtuple
|
||||
from typing import Any, Optional
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -136,10 +136,10 @@ class Qwen3OmniCodePredictorAttention(nn.Module):
|
||||
causal_mask: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
past_key_values: Optional[Cache] = None,
|
||||
cache_position: Optional[torch.LongTensor] = None,
|
||||
past_key_values: Cache | None = None,
|
||||
cache_position: torch.LongTensor | None = None,
|
||||
use_cache: bool = False,
|
||||
position_ids: Optional[torch.LongTensor] = None,
|
||||
position_ids: torch.LongTensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
bsz, seq_len, _ = hidden_states.shape
|
||||
|
||||
@@ -253,8 +253,8 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
|
||||
prefix: str,
|
||||
model_config: ModelConfig,
|
||||
layer_idx: int,
|
||||
cache_config: Optional[CacheConfig] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.layer_idx = layer_idx
|
||||
@@ -284,10 +284,10 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
|
||||
causal_mask: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
past_key_values: Optional[Cache] = None,
|
||||
cache_position: Optional[torch.LongTensor] = None,
|
||||
past_key_values: Cache | None = None,
|
||||
cache_position: torch.LongTensor | None = None,
|
||||
use_cache: bool = False,
|
||||
position_ids: Optional[torch.LongTensor] = None,
|
||||
position_ids: torch.LongTensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Self-attention with residual
|
||||
residual = hidden_states
|
||||
@@ -310,7 +310,7 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> torch.Tensor:
|
||||
assert inputs_embeds is not None, "inputs_embeds required for MTP"
|
||||
@@ -396,11 +396,11 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
inputs_embeds: torch.Tensor,
|
||||
attention_mask: Optional[torch.Tensor] = None,
|
||||
position_ids: Optional[torch.LongTensor] = None,
|
||||
past_key_values: Optional[Any] = None,
|
||||
use_cache: Optional[bool] = False,
|
||||
cache_position: Optional[torch.LongTensor] = None,
|
||||
attention_mask: torch.Tensor | None = None,
|
||||
position_ids: torch.LongTensor | None = None,
|
||||
past_key_values: Any | None = None,
|
||||
use_cache: bool | None = False,
|
||||
cache_position: torch.LongTensor | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""
|
||||
|
||||
Reference in new issue
Block a user