[model, omni] feat: Qwen3-Omni Thinker LoRA for RL training (#3915)

Signed-off-by: Qing An <qa4@rice.edu>
Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
Qing AnandSamit authored and GitHub committed 2026-06-01 00:10:14 +08:00
1 parent 5dfdf584dd
commit 57e5f0fe69
2 files changed
+27 -3

No files matched your search

@@ -64,6 +64,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
SupportsPP,
@@ -1071,6 +1072,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
nn.Module,
SupportsMultiModal,
SupportsPP,
SupportsLoRA,
SupportsMRoPE,
Qwen3OmniMoeConditionalGenerationMixin,
SupportsTranscription,
@@ -1089,6 +1091,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
"k_proj",
"v_proj",
],
"attn.qkv": [
"attn.q",
"attn.k",
"attn.v",
],
"gate_up_proj": [
"gate_proj",
"up_proj",
@@ -1111,7 +1118,8 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.vllm_config = vllm_config # needed for torch compile forward context
thinker_config: Qwen3OmniMoeThinkerConfig = vllm_config.model_config.hf_config
hf_config = vllm_config.model_config.hf_config
thinker_config: Qwen3OmniMoeThinkerConfig = getattr(hf_config, "thinker_config", None) or hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
self.config = thinker_config
@@ -1660,6 +1668,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
self,
input_tokens: list[int],
mm_features: list[MultiModalFeatureSpec],
**kwargs,
) -> tuple[torch.Tensor, int]:
"""Compute M-RoPE input positions using mm_features directly."""
seq_len = len(input_tokens)