Compare commits

..
Author SHA1 Message Date
hsliuustc0106 ade3bb27ea [Docs] Clarify Hunyuan expert remapping scope
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
2026-09-03 23:02:46 +00:00
hsliuustc0106 cecaa6344d Fix Hunyuan Image-3 expert mapping contract
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
2026-09-03 17:05:10 +00:00
14 changed files with 126 additions and 81 deletions

No files matched your search

+1
View File
@@ -3,6 +3,7 @@
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_FLASHINFER_MOE_FP16: 0
HF_HUB_DOWNLOAD_TIMEOUT: 300
HF_HUB_ETAG_TIMEOUT: 60
+1
View File
@@ -4,6 +4,7 @@
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_FLASHINFER_MOE_FP16: 0
HF_HUB_DOWNLOAD_TIMEOUT: 300
HF_HUB_ETAG_TIMEOUT: 60
+1
View File
@@ -3,6 +3,7 @@
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_FLASHINFER_MOE_FP16: 0
HF_HUB_DOWNLOAD_TIMEOUT: 300
HF_HUB_ETAG_TIMEOUT: 60
+1
View File
@@ -3,6 +3,7 @@
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_FLASHINFER_MOE_FP16: 0
HF_HUB_DOWNLOAD_TIMEOUT: 300
HF_HUB_ETAG_TIMEOUT: 60
@@ -410,8 +410,8 @@ Notes:
- `stages[].max_num_seqs` controls stage-level batching.
- Thinker/Talker commonly use `enforce_eager: false` for CUDA Graph paths.
- Code2Wav often remains eager (`enforce_eager: true`) depending on runtime behavior.
- The Qwen3-Omni deploy config pins `moe_backend: triton` for the Thinker and Talker stages. Triton has been more
stable and faster than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases.
- Qwen3-Omni defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. The Triton has been more stable & faster
than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases.
#### 2) Enable async chunk
@@ -17,9 +17,9 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
The default deployment configuration situated at `vllm_omni/deploy/qwen3_omni_moe.yaml` is resolved and loaded
automatically via the model registry, obviating the necessity for the `--deploy-config` flag in standard deployment topologies.
The bundled Qwen3-Omni setup explicitly sets `moe_backend: triton` for the Thinker and Talker stages. This keeps
both stages on vLLM's Triton unquantized MoE path and avoids the unstable FlashInfer CUTLASS unquantized MoE path on
some GPUs.
The bundled Qwen3-Omni setup defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. This keeps the Thinker & Talker on vLLM's
Triton unquantized MoE path and avoids the performance regression observed with the FlashInfer CUTLASS unquantized MoE
backend.
Asynchronous chunk streaming is **enabled by default** within the bundled configuration.
To explicitly utilize a custom deployment YAML, specify the configuration path:
@@ -87,9 +87,15 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
--stage-overrides '{"1": {"gpu_memory_utilization": 0.5}}'
```
To experiment with another supported MoE backend, set `moe_backend` in the
deployment YAML for both the Thinker and Talker stages. For the stage-based
CLI, pass `--moe-backend` directly on each stage command.
To experiment with the FlashInfer FP16 MoE path, set `VLLM_USE_FLASHINFER_MOE_FP16=1` before launching the server:
```bash
VLLM_USE_FLASHINFER_MOE_FP16=1 \
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
```
For the stage-based CLI, you usually do **not** need `--stage-overrides` for
that kind of change. Since each command launches one stage, just pass the knob
directly on that stage command:
```bash
CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+2 -2
View File
@@ -167,7 +167,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
EnvironmentVariableCategory.PUBLIC_OMNI: 24,
EnvironmentVariableCategory.INHERITED_VLLM: 20,
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
EnvironmentVariableCategory.MODEL_SPECIFIC: 57,
EnvironmentVariableCategory.MODEL_SPECIFIC: 58,
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
EnvironmentVariableCategory.INTERNAL: 2,
}
@@ -182,7 +182,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 6,
ModelEnvironmentVariableDisposition.EXTERNAL: 0,
ModelEnvironmentVariableDisposition.INTERNALIZE: 11,
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 5,
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 6,
}
+1 -21
View File
@@ -104,6 +104,7 @@ _OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS = frozenset(
def _stable_engine_arg_environment(monkeypatch, tmp_path):
from vllm_omni import platforms
monkeypatch.delenv("VLLM_USE_FLASHINFER_MOE_FP16", raising=False)
xcodec_model = tmp_path / "xcodec-model"
xcodec_model.mkdir()
monkeypatch.setenv("XCODEC1_PATH", str(xcodec_model))
@@ -136,25 +137,6 @@ def _stable_engine_arg_environment(monkeypatch, tmp_path):
monkeypatch.setitem(sys.modules, worker_module.__name__, worker_module)
def test_qwen3_omni_defaults_to_triton_moe_backend():
engine_args = {"model_arch": "Qwen3OmniMoeForConditionalGeneration"}
stage_init_utils._maybe_set_qwen3_omni_moe_backend(engine_args)
assert engine_args["moe_backend"] == "triton"
def test_qwen3_omni_preserves_explicit_moe_backend():
engine_args = {
"model_arch": "Qwen3OmniMoeForConditionalGeneration",
"moe_backend": "flashinfer",
}
stage_init_utils._maybe_set_qwen3_omni_moe_backend(engine_args)
assert engine_args["moe_backend"] == "flashinfer"
def _engine_arg_inputs(tmp_path: Path) -> tuple[PipelineConfig, DeployConfig, str]:
parent_model = tmp_path / "parent-model"
stage_model = tmp_path / "stage-model"
@@ -535,8 +517,6 @@ def test_typed_engine_args_preserve_explicit_backend_default_overrides(tmp_path)
assert {name: legacy_args[name] for name in expected} == expected
assert {name: typed_args[name] for name in expected} == expected
assert legacy_args["moe_backend"] == "triton"
assert typed_args["moe_backend"] == "triton"
@pytest.mark.parametrize("stage_scoped", [False, True], ids=["global", "stage-scoped"])
@@ -67,8 +67,11 @@ class _FakeHunyuanModel(_FakeModel):
use_cla=False,
)
def get_expert_mapping(self) -> tuple[list[object], dict[str, object]]:
return [], {}
def get_expert_mapping(self) -> list[object]:
return []
def _get_expert_weights_remapping(self) -> dict[str, object]:
return {}
def _split_qkv_weight(self, weight: torch.Tensor) -> torch.Tensor:
return weight
@@ -81,6 +84,49 @@ def test_model_loaders_do_not_call_removed_get_cache_scale(rel_path: str) -> Non
assert ".get_cache_scale(" not in source
@pytest.mark.parametrize(
("module_name", "class_name"),
[
(
"vllm_omni.model_executor.models.hunyuan_image3.hunyuan_image3",
"HunyuanModel",
),
(
"vllm_omni.diffusion.models.hunyuan_image3.hunyuan_image3_transformer",
"HunyuanImage3Model",
),
],
)
def test_hunyuan_expert_mapping_separates_public_and_local_contracts(
module_name: str,
class_name: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The public expert mapping must be a flat list, not a loader-side pair."""
module = __import__(module_name, fromlist=[class_name])
expected_mapping = [("param", "weight", 0, "gate_proj")]
fake_model = SimpleNamespace(
config=SimpleNamespace(num_experts=1),
num_redundant_experts=0,
)
monkeypatch.setattr(module, "_is_moe", lambda config: True)
if hasattr(module, "fused_moe_make_expert_params_mapping"):
monkeypatch.setattr(module, "fused_moe_make_expert_params_mapping", lambda **kwargs: expected_mapping)
else:
monkeypatch.setattr(
module,
"FusedMoE",
SimpleNamespace(make_expert_params_mapping=lambda *args, **kwargs: expected_mapping),
)
model_cls = getattr(module, class_name)
assert model_cls.get_expert_mapping(fake_model) == expected_mapping
assert model_cls._get_expert_weights_remapping(fake_model) == {
"gate_proj": ("gate_and_up_proj", 1, 2),
"up_proj": ("gate_and_up_proj", 0, 2),
}
@pytest.mark.parametrize(
("module_name", "class_name", "fake_model_cls"),
[
@@ -203,6 +203,7 @@ _MODEL_DEPRECATE_REMOVE = (
"HIGGS_AUDIO_V2_TOKENIZER_PATH",
"VLLM_OMNI_LINGBOT_ACTION_ROOT",
"VLLM_OMNI_VOXCPM_CODE_PATH",
"VLLM_USE_FLASHINFER_MOE_FP16",
"XCODEC1_PATH",
"model_stage",
)
-2
View File
@@ -27,7 +27,6 @@ stages:
max_num_batched_tokens: 32768
max_num_seqs: 64
gpu_memory_utilization: 0.9
moe_backend: triton
trust_remote_code: true
enable_prefix_caching: false
devices: "0"
@@ -39,7 +38,6 @@ stages:
max_num_batched_tokens: 32768
max_num_seqs: 64
gpu_memory_utilization: 0.6
moe_backend: triton
trust_remote_code: true
enable_prefix_caching: false
devices: "1"
@@ -2190,24 +2190,28 @@ class HunyuanImage3Model(nn.Module):
return torch.concat((q, k, v))
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
if _is_moe(self.config):
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
fused_moe_expert_mapping = FusedMoE.make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
num_redundant_experts=self.num_redundant_experts,
)
expert_weights_remapping = {
"gate_proj": ("gate_and_up_proj", 1, 2),
"up_proj": ("gate_and_up_proj", 0, 2),
}
return fused_moe_expert_mapping, expert_weights_remapping
else:
return [], {}
if not _is_moe(self.config):
return []
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
return FusedMoE.make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
num_redundant_experts=self.num_redundant_experts,
)
def _get_expert_weights_remapping(self) -> dict[str, tuple[str, int, int]]:
"""Return the vLLM-Omni-local remapping for Hunyuan checkpoint weights."""
if not _is_moe(self.config):
return {}
return {
"gate_proj": ("gate_and_up_proj", 1, 2),
"up_proj": ("gate_and_up_proj", 0, 2),
}
# rename for delay load
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
@@ -2243,7 +2247,8 @@ class HunyuanImage3Model(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping, expert_weights_remapping = self.get_expert_mapping()
expert_params_mapping = self.get_expert_mapping()
expert_weights_remapping = self._get_expert_weights_remapping()
# List of unexpected keywords in weight names
unexpected_keywords = [
+7 -7
View File
@@ -763,14 +763,13 @@ def prepare_engine_environment() -> None:
pass
def _maybe_set_qwen3_omni_moe_backend(engine_args_dict: dict[str, Any]) -> None:
"""Choose the stable MoE backend when Qwen3-Omni has no explicit choice."""
def _maybe_set_qwen3_omni_moe_env(engine_args_dict: dict[str, Any]) -> None:
if (
engine_args_dict.get("model_arch") == "Qwen3OmniMoeForConditionalGeneration"
and engine_args_dict.get("moe_backend", "auto") == "auto"
and "VLLM_USE_FLASHINFER_MOE_FP16" not in os.environ
):
engine_args_dict["moe_backend"] = "triton"
logger.info("[stage_init] Set moe_backend=triton for Qwen3-Omni stage")
os.environ["VLLM_USE_FLASHINFER_MOE_FP16"] = "0"
logger.info("[stage_init] Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni stage")
def split_devices_for_replicas(
@@ -1281,8 +1280,9 @@ def _finalize_engine_args_dict(
engine_args_dict["has_sampling_extra_args"] = has_sampling_extra_args
engine_args_dict["sampling_extra_args_keys"] = sampling_extra_args_keys
# Select the typed backend option during stage argument finalization.
_maybe_set_qwen3_omni_moe_backend(engine_args_dict)
# TODO: Remove this after the performance regression is fixed
# Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni to avoid performance regression
_maybe_set_qwen3_omni_moe_env(engine_args_dict)
return engine_args_dict
@@ -128,25 +128,29 @@ class HunyuanModel(HunYuanModel):
v = v.reshape(-1, hidden_size)
return torch.concat((q, k, v))
def get_expert_mapping(self) -> tuple[list[tuple[str, str, int, str]], dict[str, tuple[str, int, int]]]:
if _is_moe(self.config):
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
fused_moe_expert_mapping = fused_moe_make_expert_params_mapping(
model=self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
num_redundant_experts=self.num_redundant_experts,
)
expert_weights_remapping = {
"gate_proj": ("gate_and_up_proj", 1, 2),
"up_proj": ("gate_and_up_proj", 0, 2),
}
return fused_moe_expert_mapping, expert_weights_remapping
else:
return [], {}
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
if not _is_moe(self.config):
return []
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
return fused_moe_make_expert_params_mapping(
model=self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
num_redundant_experts=self.num_redundant_experts,
)
def _get_expert_weights_remapping(self) -> dict[str, tuple[str, int, int]]:
"""Return the vLLM-Omni-local remapping for Hunyuan checkpoint weights."""
if not _is_moe(self.config):
return {}
return {
"gate_proj": ("gate_and_up_proj", 1, 2),
"up_proj": ("gate_and_up_proj", 0, 2),
}
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
cla_factor = _get_cla_factor(self.config)
@@ -174,7 +178,8 @@ class HunyuanModel(HunYuanModel):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping, expert_weights_remapping = self.get_expert_mapping()
expert_params_mapping = self.get_expert_mapping()
expert_weights_remapping = self._get_expert_weights_remapping()
# List of unexpected keywords in weight names
unexpected_keywords = [