Compare commits

..
Author SHA1 Message Date
hsliuustc0106 52c8f8dbc8 Support one-sided MoT biases
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
2026-09-03 17:05:57 +00:00
14 changed files with 201 additions and 50 deletions

No files matched your search

+1
View File
@@ -3,6 +3,7 @@
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_FLASHINFER_MOE_FP16: 0
HF_HUB_DOWNLOAD_TIMEOUT: 300
HF_HUB_ETAG_TIMEOUT: 60
+1
View File
@@ -4,6 +4,7 @@
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_FLASHINFER_MOE_FP16: 0
HF_HUB_DOWNLOAD_TIMEOUT: 300
HF_HUB_ETAG_TIMEOUT: 60
+1
View File
@@ -3,6 +3,7 @@
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_FLASHINFER_MOE_FP16: 0
HF_HUB_DOWNLOAD_TIMEOUT: 300
HF_HUB_ETAG_TIMEOUT: 60
+1
View File
@@ -3,6 +3,7 @@
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_FLASHINFER_MOE_FP16: 0
HF_HUB_DOWNLOAD_TIMEOUT: 300
HF_HUB_ETAG_TIMEOUT: 60
@@ -410,8 +410,8 @@ Notes:
- `stages[].max_num_seqs` controls stage-level batching.
- Thinker/Talker commonly use `enforce_eager: false` for CUDA Graph paths.
- Code2Wav often remains eager (`enforce_eager: true`) depending on runtime behavior.
- The Qwen3-Omni deploy config pins `moe_backend: triton` for the Thinker and Talker stages. Triton has been more
stable and faster than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases.
- Qwen3-Omni defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. The Triton has been more stable & faster
than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases.
#### 2) Enable async chunk
@@ -17,9 +17,9 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
The default deployment configuration situated at `vllm_omni/deploy/qwen3_omni_moe.yaml` is resolved and loaded
automatically via the model registry, obviating the necessity for the `--deploy-config` flag in standard deployment topologies.
The bundled Qwen3-Omni setup explicitly sets `moe_backend: triton` for the Thinker and Talker stages. This keeps
both stages on vLLM's Triton unquantized MoE path and avoids the unstable FlashInfer CUTLASS unquantized MoE path on
some GPUs.
The bundled Qwen3-Omni setup defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. This keeps the Thinker & Talker on vLLM's
Triton unquantized MoE path and avoids the performance regression observed with the FlashInfer CUTLASS unquantized MoE
backend.
Asynchronous chunk streaming is **enabled by default** within the bundled configuration.
To explicitly utilize a custom deployment YAML, specify the configuration path:
@@ -87,9 +87,15 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
--stage-overrides '{"1": {"gpu_memory_utilization": 0.5}}'
```
To experiment with another supported MoE backend, set `moe_backend` in the
deployment YAML for both the Thinker and Talker stages. For the stage-based
CLI, pass `--moe-backend` directly on each stage command.
To experiment with the FlashInfer FP16 MoE path, set `VLLM_USE_FLASHINFER_MOE_FP16=1` before launching the server:
```bash
VLLM_USE_FLASHINFER_MOE_FP16=1 \
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
```
For the stage-based CLI, you usually do **not** need `--stage-overrides` for
that kind of change. Since each command launches one stage, just pass the knob
directly on that stage command:
```bash
CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+2 -2
View File
@@ -167,7 +167,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
EnvironmentVariableCategory.PUBLIC_OMNI: 24,
EnvironmentVariableCategory.INHERITED_VLLM: 20,
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
EnvironmentVariableCategory.MODEL_SPECIFIC: 57,
EnvironmentVariableCategory.MODEL_SPECIFIC: 58,
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
EnvironmentVariableCategory.INTERNAL: 2,
}
@@ -182,7 +182,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 6,
ModelEnvironmentVariableDisposition.EXTERNAL: 0,
ModelEnvironmentVariableDisposition.INTERNALIZE: 11,
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 5,
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 6,
}
@@ -0,0 +1,95 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import pytest
import torch
from vllm_omni.diffusion.layers.mot.ops import mot_gemm
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
_CONFIG = {
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 64,
"BLOCK_SIZE_K": 32,
"GROUP_SIZE_M": 8,
"num_warps": 4,
"num_stages": 2,
}
class _RecordingKernel:
def __init__(self) -> None:
self.calls: list[tuple[tuple[object, ...], dict[str, object]]] = []
def __getitem__(self, grid):
del grid
return self._launch
def _launch(self, *args, **kwargs) -> None:
self.calls.append((args, kwargs))
@pytest.mark.parametrize(
("bias_text", "bias_vae"),
[(None, None), ("text", None), (None, "vae"), ("text", "vae")],
ids=["no-bias", "text-only", "vae-only", "both"],
)
def test_invoke_mot_gemm_accepts_independent_biases(
bias_text: str | None,
bias_vae: str | None,
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _RecordingKernel()
monkeypatch.setattr(mot_gemm, "mot_unified_gemm_kernel", kernel)
A = torch.ones((3, 4), dtype=torch.float16)
B_text = torch.ones((4, 5), dtype=torch.float16)
B_vae = torch.ones((4, 5), dtype=torch.float16)
C = torch.empty((3, 5), dtype=torch.float16)
text_indices = torch.tensor([0, 2], dtype=torch.long)
vae_indices = torch.tensor([1], dtype=torch.long)
text_bias = torch.ones(5, dtype=torch.float16) if bias_text else None
vae_bias = torch.ones(5, dtype=torch.float16) if bias_vae else None
mot_gemm.invoke_mot_gemm(
A=A,
B_text=B_text,
B_vae=B_vae,
C=C,
bias_text=text_bias,
bias_vae=vae_bias,
text_indices=text_indices,
vae_indices=vae_indices,
A_scale=None,
B_text_scale=None,
B_vae_scale=None,
use_fp8_w8a8=False,
use_int8_w8a8=False,
use_int8_w8a16=False,
use_int4_w4a16=False,
A_per_channel_quant=False,
B_per_channel_quant=False,
config=_CONFIG,
)
assert len(kernel.calls) == 1
args, kwargs = kernel.calls[0]
assert kwargs["HAS_BIAS_TEXT"] is (text_bias is not None)
assert kwargs["HAS_BIAS_VAE"] is (vae_bias is not None)
assert "HAS_BIAS" not in kwargs
launched_text_bias, launched_vae_bias = args[4:6]
if text_bias is None and vae_bias is None:
assert launched_text_bias == launched_vae_bias == 0
elif text_bias is None:
assert launched_text_bias is vae_bias
assert launched_vae_bias is vae_bias
elif vae_bias is None:
assert launched_text_bias is text_bias
assert launched_vae_bias is text_bias
else:
assert launched_text_bias is text_bias
assert launched_vae_bias is vae_bias
@@ -513,6 +513,57 @@ def test_mot_o_proj(
)
@pytest.mark.parametrize(
("bias_text", "bias_vae"),
[(True, False), (False, True)],
ids=["text-only", "vae-only"],
)
def test_mot_row_one_sided_bias(bias_text: bool, bias_vae: bool):
"""The fused row projection must support a bias on only one expert."""
dcfg = _parse_dtype("w16a16_bf16")
K = 32
N = 64
M = 5
with set_current_vllm_config(VllmConfig()):
text_linear = RowParallelLinear(
K,
N,
bias=bias_text,
input_is_parallel=True,
params_dtype=dcfg.torch_dtype,
disable_tp=True,
).cuda()
vae_linear = RowParallelLinear(
K,
N,
bias=bias_vae,
input_is_parallel=True,
params_dtype=dcfg.torch_dtype,
disable_tp=True,
).cuda()
mot_linear = MoTRowParallelLinear(
K,
N,
bias=bias_text,
vae_bias=bias_vae,
input_is_parallel=True,
params_dtype=dcfg.torch_dtype,
disable_tp=True,
).cuda()
_sync_weights(text_linear, vae_linear, mot_linear)
text_idx = torch.tensor([0, 2], dtype=torch.long, device="cuda")
vae_idx = torch.tensor([1, 3, 4], dtype=torch.long, device="cuda")
x = torch.randn(M, K, dtype=dcfg.torch_dtype, device="cuda")
with torch.no_grad():
ref = _reference_forward(x, text_idx, vae_idx, text_linear, vae_linear)
mot_out, _ = mot_linear(x, text_idx, vae_idx)
_check_and_report(ref, mot_out, f"Row one-sided bias text={bias_text} vae={bias_vae}")
# =========================================================================
# Test: und-mode (text_indices=None) — falls back to standard forward
# =========================================================================
+1 -21
View File
@@ -104,6 +104,7 @@ _OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS = frozenset(
def _stable_engine_arg_environment(monkeypatch, tmp_path):
from vllm_omni import platforms
monkeypatch.delenv("VLLM_USE_FLASHINFER_MOE_FP16", raising=False)
xcodec_model = tmp_path / "xcodec-model"
xcodec_model.mkdir()
monkeypatch.setenv("XCODEC1_PATH", str(xcodec_model))
@@ -136,25 +137,6 @@ def _stable_engine_arg_environment(monkeypatch, tmp_path):
monkeypatch.setitem(sys.modules, worker_module.__name__, worker_module)
def test_qwen3_omni_defaults_to_triton_moe_backend():
engine_args = {"model_arch": "Qwen3OmniMoeForConditionalGeneration"}
stage_init_utils._maybe_set_qwen3_omni_moe_backend(engine_args)
assert engine_args["moe_backend"] == "triton"
def test_qwen3_omni_preserves_explicit_moe_backend():
engine_args = {
"model_arch": "Qwen3OmniMoeForConditionalGeneration",
"moe_backend": "flashinfer",
}
stage_init_utils._maybe_set_qwen3_omni_moe_backend(engine_args)
assert engine_args["moe_backend"] == "flashinfer"
def _engine_arg_inputs(tmp_path: Path) -> tuple[PipelineConfig, DeployConfig, str]:
parent_model = tmp_path / "parent-model"
stage_model = tmp_path / "stage-model"
@@ -535,8 +517,6 @@ def test_typed_engine_args_preserve_explicit_backend_default_overrides(tmp_path)
assert {name: legacy_args[name] for name in expected} == expected
assert {name: typed_args[name] for name in expected} == expected
assert legacy_args["moe_backend"] == "triton"
assert typed_args["moe_backend"] == "triton"
@pytest.mark.parametrize("stage_scoped", [False, True], ids=["global", "stage-scoped"])
@@ -203,6 +203,7 @@ _MODEL_DEPRECATE_REMOVE = (
"HIGGS_AUDIO_V2_TOKENIZER_PATH",
"VLLM_OMNI_LINGBOT_ACTION_ROOT",
"VLLM_OMNI_VOXCPM_CODE_PATH",
"VLLM_USE_FLASHINFER_MOE_FP16",
"XCODEC1_PATH",
"model_stage",
)
-2
View File
@@ -27,7 +27,6 @@ stages:
max_num_batched_tokens: 32768
max_num_seqs: 64
gpu_memory_utilization: 0.9
moe_backend: triton
trust_remote_code: true
enable_prefix_caching: false
devices: "0"
@@ -39,7 +38,6 @@ stages:
max_num_batched_tokens: 32768
max_num_seqs: 64
gpu_memory_utilization: 0.6
moe_backend: triton
trust_remote_code: true
enable_prefix_caching: false
devices: "1"
+26 -10
View File
@@ -238,6 +238,7 @@ def _get_mot_pointers(
cur_stride_bk = stride_bk_vae
cur_stride_bn = stride_bn_vae
M_limit = M_vae
is_vae = 1
# Text Path
if pid < num_pid_m_text * num_pid_n:
@@ -251,6 +252,7 @@ def _get_mot_pointers(
cur_stride_bk = stride_bk_text
cur_stride_bn = stride_bn_text
M_limit = M_text
is_vae = 0
# 3. Calculate Grid coordinates(grouping)
cur_num_pid_m = tl.cdiv(M_limit, BLOCK_SIZE_M)
@@ -283,6 +285,7 @@ def _get_mot_pointers(
offs_n,
n_mask, # N-dim info
M_limit, # Boundary
is_vae, # Selected expert kind: 0=text, 1=VAE
cur_b_ptr,
cur_bias_ptr, # Selected Pointers
cur_scale_b_ptr, # Selected Scale Pointer
@@ -533,7 +536,8 @@ def mot_unified_gemm_kernel(
# Quant Control
# 0=None, 1=W8A8, 2=W8A16, 3=W4A16
QUANT_TYPE: tl.constexpr,
HAS_BIAS: tl.constexpr = False,
HAS_BIAS_TEXT: tl.constexpr = False,
HAS_BIAS_VAE: tl.constexpr = False,
):
pid = tl.program_id(axis=0)
@@ -548,6 +552,7 @@ def mot_unified_gemm_kernel(
offs_n,
n_mask,
M_limit,
is_vae,
cur_b_ptr,
cur_bias_ptr,
cur_scale_b_ptr,
@@ -675,8 +680,12 @@ def mot_unified_gemm_kernel(
# -----------------------------------------------------------
# Bias Add
if HAS_BIAS:
if HAS_BIAS_TEXT or HAS_BIAS_VAE:
bias = tl.load(cur_bias_ptr + offs_n, mask=n_mask, other=0.0)
if HAS_BIAS_TEXT and not HAS_BIAS_VAE:
bias = tl.where(is_vae == 0, bias, 0.0)
elif HAS_BIAS_VAE and not HAS_BIAS_TEXT:
bias = tl.where(is_vae == 1, bias, 0.0)
c = c + bias[None, :] # reshape to (1, BLOCK_SIZE_N)
# Cast C into the output dtype
@@ -832,11 +841,8 @@ def invoke_mot_gemm(
STRIDE_BK_IS_1 = (B_text.stride(0) == 1) and (B_vae.stride(0) == 1)
STRIDE_BN_IS_1 = (B_text.stride(1) == 1) and (B_vae.stride(1) == 1)
# bias check
assert (bias_text is None) == (bias_vae is None), (
"Bias must be provided for both Text and VAE simultaneously, or neither."
)
has_bias = bias_text is not None
has_bias_text = bias_text is not None
has_bias_vae = bias_vae is not None
# --- 3. Grid Calculation ---
def grid(META):
@@ -853,7 +859,8 @@ def invoke_mot_gemm(
"ACCUMULATOR_DTYPE": ACCUMULATOR_DTYPE,
"COMPUTE_DTYPE": COMPUTE_DTYPE,
"OUTPUT_DTYPE": OUTPUT_DTYPE,
"HAS_BIAS": has_bias,
"HAS_BIAS_TEXT": has_bias_text,
"HAS_BIAS_VAE": has_bias_vae,
"EVEN_K": EVEN_K,
"EVEN_N": EVEN_N,
"STRIDE_AK_IS_1": STRIDE_AK_IS_1,
@@ -866,8 +873,17 @@ def invoke_mot_gemm(
p_a_scale = A_scale if A_scale is not None else 0
p_b_text_scale = B_text_scale if B_text_scale is not None else 0
p_b_vae_scale = B_vae_scale if B_vae_scale is not None else 0
p_bias_text = bias_text if bias_text is not None else 0
p_bias_vae = bias_vae if bias_vae is not None else 0
# Triton requires both route-selected bias arguments to have a pointer
# type. Alias the missing route to the available tensor; its compile-time
# bias flag and runtime route mask keep it from being read for that route.
if bias_text is None:
p_bias_text = bias_vae if bias_vae is not None else 0
else:
p_bias_text = bias_text
if bias_vae is None:
p_bias_vae = bias_text if bias_text is not None else 0
else:
p_bias_vae = bias_vae
# Quantization granularity
stride_scale_a = 1 if A_per_channel_quant else 0
+7 -7
View File
@@ -763,14 +763,13 @@ def prepare_engine_environment() -> None:
pass
def _maybe_set_qwen3_omni_moe_backend(engine_args_dict: dict[str, Any]) -> None:
"""Choose the stable MoE backend when Qwen3-Omni has no explicit choice."""
def _maybe_set_qwen3_omni_moe_env(engine_args_dict: dict[str, Any]) -> None:
if (
engine_args_dict.get("model_arch") == "Qwen3OmniMoeForConditionalGeneration"
and engine_args_dict.get("moe_backend", "auto") == "auto"
and "VLLM_USE_FLASHINFER_MOE_FP16" not in os.environ
):
engine_args_dict["moe_backend"] = "triton"
logger.info("[stage_init] Set moe_backend=triton for Qwen3-Omni stage")
os.environ["VLLM_USE_FLASHINFER_MOE_FP16"] = "0"
logger.info("[stage_init] Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni stage")
def split_devices_for_replicas(
@@ -1281,8 +1280,9 @@ def _finalize_engine_args_dict(
engine_args_dict["has_sampling_extra_args"] = has_sampling_extra_args
engine_args_dict["sampling_extra_args_keys"] = sampling_extra_args_keys
# Select the typed backend option during stage argument finalization.
_maybe_set_qwen3_omni_moe_backend(engine_args_dict)
# TODO: Remove this after the performance regression is fixed
# Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni to avoid performance regression
_maybe_set_qwen3_omni_moe_env(engine_args_dict)
return engine_args_dict