Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ade3bb27ea | ||
|
|
cecaa6344d |
No files matched your search
@@ -3,6 +3,7 @@
|
||||
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 60
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 60
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 60
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 60
|
||||
|
||||
|
||||
@@ -410,8 +410,8 @@ Notes:
|
||||
- `stages[].max_num_seqs` controls stage-level batching.
|
||||
- Thinker/Talker commonly use `enforce_eager: false` for CUDA Graph paths.
|
||||
- Code2Wav often remains eager (`enforce_eager: true`) depending on runtime behavior.
|
||||
- The Qwen3-Omni deploy config pins `moe_backend: triton` for the Thinker and Talker stages. Triton has been more
|
||||
stable and faster than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases.
|
||||
- Qwen3-Omni defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. The Triton has been more stable & faster
|
||||
than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases.
|
||||
|
||||
#### 2) Enable async chunk
|
||||
|
||||
|
||||
@@ -17,9 +17,9 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
|
||||
The default deployment configuration situated at `vllm_omni/deploy/qwen3_omni_moe.yaml` is resolved and loaded
|
||||
automatically via the model registry, obviating the necessity for the `--deploy-config` flag in standard deployment topologies.
|
||||
The bundled Qwen3-Omni setup explicitly sets `moe_backend: triton` for the Thinker and Talker stages. This keeps
|
||||
both stages on vLLM's Triton unquantized MoE path and avoids the unstable FlashInfer CUTLASS unquantized MoE path on
|
||||
some GPUs.
|
||||
The bundled Qwen3-Omni setup defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. This keeps the Thinker & Talker on vLLM's
|
||||
Triton unquantized MoE path and avoids the performance regression observed with the FlashInfer CUTLASS unquantized MoE
|
||||
backend.
|
||||
Asynchronous chunk streaming is **enabled by default** within the bundled configuration.
|
||||
|
||||
To explicitly utilize a custom deployment YAML, specify the configuration path:
|
||||
@@ -87,9 +87,15 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--stage-overrides '{"1": {"gpu_memory_utilization": 0.5}}'
|
||||
```
|
||||
|
||||
To experiment with another supported MoE backend, set `moe_backend` in the
|
||||
deployment YAML for both the Thinker and Talker stages. For the stage-based
|
||||
CLI, pass `--moe-backend` directly on each stage command.
|
||||
To experiment with the FlashInfer FP16 MoE path, set `VLLM_USE_FLASHINFER_MOE_FP16=1` before launching the server:
|
||||
```bash
|
||||
VLLM_USE_FLASHINFER_MOE_FP16=1 \
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
For the stage-based CLI, you usually do **not** need `--stage-overrides` for
|
||||
that kind of change. Since each command launches one stage, just pass the knob
|
||||
directly on that stage command:
|
||||
|
||||
```bash
|
||||
CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
|
||||
|
||||
@@ -167,7 +167,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
||||
EnvironmentVariableCategory.PUBLIC_OMNI: 24,
|
||||
EnvironmentVariableCategory.INHERITED_VLLM: 20,
|
||||
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
|
||||
EnvironmentVariableCategory.MODEL_SPECIFIC: 57,
|
||||
EnvironmentVariableCategory.MODEL_SPECIFIC: 58,
|
||||
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
|
||||
EnvironmentVariableCategory.INTERNAL: 2,
|
||||
}
|
||||
@@ -182,7 +182,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
||||
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 6,
|
||||
ModelEnvironmentVariableDisposition.EXTERNAL: 0,
|
||||
ModelEnvironmentVariableDisposition.INTERNALIZE: 11,
|
||||
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 5,
|
||||
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 6,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -104,6 +104,7 @@ _OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS = frozenset(
|
||||
def _stable_engine_arg_environment(monkeypatch, tmp_path):
|
||||
from vllm_omni import platforms
|
||||
|
||||
monkeypatch.delenv("VLLM_USE_FLASHINFER_MOE_FP16", raising=False)
|
||||
xcodec_model = tmp_path / "xcodec-model"
|
||||
xcodec_model.mkdir()
|
||||
monkeypatch.setenv("XCODEC1_PATH", str(xcodec_model))
|
||||
@@ -136,25 +137,6 @@ def _stable_engine_arg_environment(monkeypatch, tmp_path):
|
||||
monkeypatch.setitem(sys.modules, worker_module.__name__, worker_module)
|
||||
|
||||
|
||||
def test_qwen3_omni_defaults_to_triton_moe_backend():
|
||||
engine_args = {"model_arch": "Qwen3OmniMoeForConditionalGeneration"}
|
||||
|
||||
stage_init_utils._maybe_set_qwen3_omni_moe_backend(engine_args)
|
||||
|
||||
assert engine_args["moe_backend"] == "triton"
|
||||
|
||||
|
||||
def test_qwen3_omni_preserves_explicit_moe_backend():
|
||||
engine_args = {
|
||||
"model_arch": "Qwen3OmniMoeForConditionalGeneration",
|
||||
"moe_backend": "flashinfer",
|
||||
}
|
||||
|
||||
stage_init_utils._maybe_set_qwen3_omni_moe_backend(engine_args)
|
||||
|
||||
assert engine_args["moe_backend"] == "flashinfer"
|
||||
|
||||
|
||||
def _engine_arg_inputs(tmp_path: Path) -> tuple[PipelineConfig, DeployConfig, str]:
|
||||
parent_model = tmp_path / "parent-model"
|
||||
stage_model = tmp_path / "stage-model"
|
||||
@@ -535,8 +517,6 @@ def test_typed_engine_args_preserve_explicit_backend_default_overrides(tmp_path)
|
||||
|
||||
assert {name: legacy_args[name] for name in expected} == expected
|
||||
assert {name: typed_args[name] for name in expected} == expected
|
||||
assert legacy_args["moe_backend"] == "triton"
|
||||
assert typed_args["moe_backend"] == "triton"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stage_scoped", [False, True], ids=["global", "stage-scoped"])
|
||||
|
||||
@@ -67,8 +67,11 @@ class _FakeHunyuanModel(_FakeModel):
|
||||
use_cla=False,
|
||||
)
|
||||
|
||||
def get_expert_mapping(self) -> tuple[list[object], dict[str, object]]:
|
||||
return [], {}
|
||||
def get_expert_mapping(self) -> list[object]:
|
||||
return []
|
||||
|
||||
def _get_expert_weights_remapping(self) -> dict[str, object]:
|
||||
return {}
|
||||
|
||||
def _split_qkv_weight(self, weight: torch.Tensor) -> torch.Tensor:
|
||||
return weight
|
||||
@@ -81,6 +84,49 @@ def test_model_loaders_do_not_call_removed_get_cache_scale(rel_path: str) -> Non
|
||||
assert ".get_cache_scale(" not in source
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module_name", "class_name"),
|
||||
[
|
||||
(
|
||||
"vllm_omni.model_executor.models.hunyuan_image3.hunyuan_image3",
|
||||
"HunyuanModel",
|
||||
),
|
||||
(
|
||||
"vllm_omni.diffusion.models.hunyuan_image3.hunyuan_image3_transformer",
|
||||
"HunyuanImage3Model",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_hunyuan_expert_mapping_separates_public_and_local_contracts(
|
||||
module_name: str,
|
||||
class_name: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The public expert mapping must be a flat list, not a loader-side pair."""
|
||||
module = __import__(module_name, fromlist=[class_name])
|
||||
expected_mapping = [("param", "weight", 0, "gate_proj")]
|
||||
fake_model = SimpleNamespace(
|
||||
config=SimpleNamespace(num_experts=1),
|
||||
num_redundant_experts=0,
|
||||
)
|
||||
monkeypatch.setattr(module, "_is_moe", lambda config: True)
|
||||
if hasattr(module, "fused_moe_make_expert_params_mapping"):
|
||||
monkeypatch.setattr(module, "fused_moe_make_expert_params_mapping", lambda **kwargs: expected_mapping)
|
||||
else:
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"FusedMoE",
|
||||
SimpleNamespace(make_expert_params_mapping=lambda *args, **kwargs: expected_mapping),
|
||||
)
|
||||
|
||||
model_cls = getattr(module, class_name)
|
||||
assert model_cls.get_expert_mapping(fake_model) == expected_mapping
|
||||
assert model_cls._get_expert_weights_remapping(fake_model) == {
|
||||
"gate_proj": ("gate_and_up_proj", 1, 2),
|
||||
"up_proj": ("gate_and_up_proj", 0, 2),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module_name", "class_name", "fake_model_cls"),
|
||||
[
|
||||
|
||||
@@ -203,6 +203,7 @@ _MODEL_DEPRECATE_REMOVE = (
|
||||
"HIGGS_AUDIO_V2_TOKENIZER_PATH",
|
||||
"VLLM_OMNI_LINGBOT_ACTION_ROOT",
|
||||
"VLLM_OMNI_VOXCPM_CODE_PATH",
|
||||
"VLLM_USE_FLASHINFER_MOE_FP16",
|
||||
"XCODEC1_PATH",
|
||||
"model_stage",
|
||||
)
|
||||
|
||||
@@ -27,7 +27,6 @@ stages:
|
||||
max_num_batched_tokens: 32768
|
||||
max_num_seqs: 64
|
||||
gpu_memory_utilization: 0.9
|
||||
moe_backend: triton
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
devices: "0"
|
||||
@@ -39,7 +38,6 @@ stages:
|
||||
max_num_batched_tokens: 32768
|
||||
max_num_seqs: 64
|
||||
gpu_memory_utilization: 0.6
|
||||
moe_backend: triton
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
devices: "1"
|
||||
|
||||
@@ -2190,24 +2190,28 @@ class HunyuanImage3Model(nn.Module):
|
||||
return torch.concat((q, k, v))
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
if _is_moe(self.config):
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
fused_moe_expert_mapping = FusedMoE.make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
expert_weights_remapping = {
|
||||
"gate_proj": ("gate_and_up_proj", 1, 2),
|
||||
"up_proj": ("gate_and_up_proj", 0, 2),
|
||||
}
|
||||
return fused_moe_expert_mapping, expert_weights_remapping
|
||||
else:
|
||||
return [], {}
|
||||
if not _is_moe(self.config):
|
||||
return []
|
||||
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
return FusedMoE.make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
def _get_expert_weights_remapping(self) -> dict[str, tuple[str, int, int]]:
|
||||
"""Return the vLLM-Omni-local remapping for Hunyuan checkpoint weights."""
|
||||
if not _is_moe(self.config):
|
||||
return {}
|
||||
return {
|
||||
"gate_proj": ("gate_and_up_proj", 1, 2),
|
||||
"up_proj": ("gate_and_up_proj", 0, 2),
|
||||
}
|
||||
|
||||
# rename for delay load
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
@@ -2243,7 +2247,8 @@ class HunyuanImage3Model(nn.Module):
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping, expert_weights_remapping = self.get_expert_mapping()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
expert_weights_remapping = self._get_expert_weights_remapping()
|
||||
|
||||
# List of unexpected keywords in weight names
|
||||
unexpected_keywords = [
|
||||
|
||||
@@ -763,14 +763,13 @@ def prepare_engine_environment() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _maybe_set_qwen3_omni_moe_backend(engine_args_dict: dict[str, Any]) -> None:
|
||||
"""Choose the stable MoE backend when Qwen3-Omni has no explicit choice."""
|
||||
def _maybe_set_qwen3_omni_moe_env(engine_args_dict: dict[str, Any]) -> None:
|
||||
if (
|
||||
engine_args_dict.get("model_arch") == "Qwen3OmniMoeForConditionalGeneration"
|
||||
and engine_args_dict.get("moe_backend", "auto") == "auto"
|
||||
and "VLLM_USE_FLASHINFER_MOE_FP16" not in os.environ
|
||||
):
|
||||
engine_args_dict["moe_backend"] = "triton"
|
||||
logger.info("[stage_init] Set moe_backend=triton for Qwen3-Omni stage")
|
||||
os.environ["VLLM_USE_FLASHINFER_MOE_FP16"] = "0"
|
||||
logger.info("[stage_init] Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni stage")
|
||||
|
||||
|
||||
def split_devices_for_replicas(
|
||||
@@ -1281,8 +1280,9 @@ def _finalize_engine_args_dict(
|
||||
engine_args_dict["has_sampling_extra_args"] = has_sampling_extra_args
|
||||
engine_args_dict["sampling_extra_args_keys"] = sampling_extra_args_keys
|
||||
|
||||
# Select the typed backend option during stage argument finalization.
|
||||
_maybe_set_qwen3_omni_moe_backend(engine_args_dict)
|
||||
# TODO: Remove this after the performance regression is fixed
|
||||
# Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni to avoid performance regression
|
||||
_maybe_set_qwen3_omni_moe_env(engine_args_dict)
|
||||
return engine_args_dict
|
||||
|
||||
|
||||
|
||||
@@ -128,25 +128,29 @@ class HunyuanModel(HunYuanModel):
|
||||
v = v.reshape(-1, hidden_size)
|
||||
return torch.concat((q, k, v))
|
||||
|
||||
def get_expert_mapping(self) -> tuple[list[tuple[str, str, int, str]], dict[str, tuple[str, int, int]]]:
|
||||
if _is_moe(self.config):
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
fused_moe_expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
model=self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
expert_weights_remapping = {
|
||||
"gate_proj": ("gate_and_up_proj", 1, 2),
|
||||
"up_proj": ("gate_and_up_proj", 0, 2),
|
||||
}
|
||||
return fused_moe_expert_mapping, expert_weights_remapping
|
||||
else:
|
||||
return [], {}
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
if not _is_moe(self.config):
|
||||
return []
|
||||
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
return fused_moe_make_expert_params_mapping(
|
||||
model=self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
def _get_expert_weights_remapping(self) -> dict[str, tuple[str, int, int]]:
|
||||
"""Return the vLLM-Omni-local remapping for Hunyuan checkpoint weights."""
|
||||
if not _is_moe(self.config):
|
||||
return {}
|
||||
return {
|
||||
"gate_proj": ("gate_and_up_proj", 1, 2),
|
||||
"up_proj": ("gate_and_up_proj", 0, 2),
|
||||
}
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
cla_factor = _get_cla_factor(self.config)
|
||||
@@ -174,7 +178,8 @@ class HunyuanModel(HunYuanModel):
|
||||
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping, expert_weights_remapping = self.get_expert_mapping()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
expert_weights_remapping = self._get_expert_weights_remapping()
|
||||
|
||||
# List of unexpected keywords in weight names
|
||||
unexpected_keywords = [
|
||||
|
||||
Reference in new issue
Block a user