Compare commits

..
Author SHA1 Message Date
hsliuustc0106 0527aadf6b [Bugfix] Align Hub kernel dependency guidance
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
2026-09-03 23:02:19 +00:00
hsliuustc0106 6b95de36a6 Update Hub kernel dependency floor
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
2026-09-03 17:04:25 +00:00
15 changed files with 35 additions and 45 deletions

No files matched your search

+1
View File
@@ -3,6 +3,7 @@
env: env:
VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_FLASHINFER_MOE_FP16: 0
HF_HUB_DOWNLOAD_TIMEOUT: 300 HF_HUB_DOWNLOAD_TIMEOUT: 300
HF_HUB_ETAG_TIMEOUT: 60 HF_HUB_ETAG_TIMEOUT: 60
+1
View File
@@ -4,6 +4,7 @@
env: env:
VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_FLASHINFER_MOE_FP16: 0
HF_HUB_DOWNLOAD_TIMEOUT: 300 HF_HUB_DOWNLOAD_TIMEOUT: 300
HF_HUB_ETAG_TIMEOUT: 60 HF_HUB_ETAG_TIMEOUT: 60
+1
View File
@@ -3,6 +3,7 @@
env: env:
VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_FLASHINFER_MOE_FP16: 0
HF_HUB_DOWNLOAD_TIMEOUT: 300 HF_HUB_DOWNLOAD_TIMEOUT: 300
HF_HUB_ETAG_TIMEOUT: 60 HF_HUB_ETAG_TIMEOUT: 60
+1
View File
@@ -3,6 +3,7 @@
env: env:
VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_WORKER_MULTIPROC_METHOD: spawn
VLLM_USE_FLASHINFER_MOE_FP16: 0
HF_HUB_DOWNLOAD_TIMEOUT: 300 HF_HUB_DOWNLOAD_TIMEOUT: 300
HF_HUB_ETAG_TIMEOUT: 60 HF_HUB_ETAG_TIMEOUT: 60
@@ -410,8 +410,8 @@ Notes:
- `stages[].max_num_seqs` controls stage-level batching. - `stages[].max_num_seqs` controls stage-level batching.
- Thinker/Talker commonly use `enforce_eager: false` for CUDA Graph paths. - Thinker/Talker commonly use `enforce_eager: false` for CUDA Graph paths.
- Code2Wav often remains eager (`enforce_eager: true`) depending on runtime behavior. - Code2Wav often remains eager (`enforce_eager: true`) depending on runtime behavior.
- The Qwen3-Omni deploy config pins `moe_backend: triton` for the Thinker and Talker stages. Triton has been more - Qwen3-Omni defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. The Triton has been more stable & faster
stable and faster than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases. than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases.
#### 2) Enable async chunk #### 2) Enable async chunk
@@ -12,7 +12,7 @@ implementations to reduce numerical drift and sampling divergence.
## Installation ## Installation
```bash ```bash
pip install kernels==0.14.1 pip install kernels==0.15.2
``` ```
If `kernels` is unavailable, an explicit `FLASH_ATTN_HUB` or If `kernels` is unavailable, an explicit `FLASH_ATTN_HUB` or
@@ -17,9 +17,9 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
The default deployment configuration situated at `vllm_omni/deploy/qwen3_omni_moe.yaml` is resolved and loaded The default deployment configuration situated at `vllm_omni/deploy/qwen3_omni_moe.yaml` is resolved and loaded
automatically via the model registry, obviating the necessity for the `--deploy-config` flag in standard deployment topologies. automatically via the model registry, obviating the necessity for the `--deploy-config` flag in standard deployment topologies.
The bundled Qwen3-Omni setup explicitly sets `moe_backend: triton` for the Thinker and Talker stages. This keeps The bundled Qwen3-Omni setup defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. This keeps the Thinker & Talker on vLLM's
both stages on vLLM's Triton unquantized MoE path and avoids the unstable FlashInfer CUTLASS unquantized MoE path on Triton unquantized MoE path and avoids the performance regression observed with the FlashInfer CUTLASS unquantized MoE
some GPUs. backend.
Asynchronous chunk streaming is **enabled by default** within the bundled configuration. Asynchronous chunk streaming is **enabled by default** within the bundled configuration.
To explicitly utilize a custom deployment YAML, specify the configuration path: To explicitly utilize a custom deployment YAML, specify the configuration path:
@@ -87,9 +87,15 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
--stage-overrides '{"1": {"gpu_memory_utilization": 0.5}}' --stage-overrides '{"1": {"gpu_memory_utilization": 0.5}}'
``` ```
To experiment with another supported MoE backend, set `moe_backend` in the To experiment with the FlashInfer FP16 MoE path, set `VLLM_USE_FLASHINFER_MOE_FP16=1` before launching the server:
deployment YAML for both the Thinker and Talker stages. For the stage-based ```bash
CLI, pass `--moe-backend` directly on each stage command. VLLM_USE_FLASHINFER_MOE_FP16=1 \
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
```
For the stage-based CLI, you usually do **not** need `--stage-overrides` for
that kind of change. Since each command launches one stage, just pass the knob
directly on that stage command:
```bash ```bash
CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+3 -2
View File
@@ -13,8 +13,9 @@ transformers >= 5.10.1, < 5.15
av>=14.0.0 av>=14.0.0
omegaconf>=2.3.0 omegaconf>=2.3.0
diffusers==0.40.0 diffusers==0.40.0
# Hub kernels, including LTX-2.5 NATTEN # Hub kernels, including LTX-2.5 NATTEN. Keep this pin in sync with the
kernels==0.14.1 # supported transformers range above.
kernels==0.15.2
safetensors>=0.8.0 safetensors>=0.8.0
accelerate==1.12.0 accelerate==1.12.0
soundfile>=0.13.1 soundfile>=0.13.1
+2 -2
View File
@@ -167,7 +167,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
EnvironmentVariableCategory.PUBLIC_OMNI: 24, EnvironmentVariableCategory.PUBLIC_OMNI: 24,
EnvironmentVariableCategory.INHERITED_VLLM: 20, EnvironmentVariableCategory.INHERITED_VLLM: 20,
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27, EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
EnvironmentVariableCategory.MODEL_SPECIFIC: 57, EnvironmentVariableCategory.MODEL_SPECIFIC: 58,
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20, EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
EnvironmentVariableCategory.INTERNAL: 2, EnvironmentVariableCategory.INTERNAL: 2,
} }
@@ -182,7 +182,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 6, ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 6,
ModelEnvironmentVariableDisposition.EXTERNAL: 0, ModelEnvironmentVariableDisposition.EXTERNAL: 0,
ModelEnvironmentVariableDisposition.INTERNALIZE: 11, ModelEnvironmentVariableDisposition.INTERNALIZE: 11,
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 5, ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 6,
} }
@@ -386,7 +386,7 @@ def test_ltx25_natten_failure_has_actionable_omni_remedy(monkeypatch):
MissingNattenProcessor, MissingNattenProcessor,
) )
with pytest.raises(RuntimeError, match=r"kernels==0\.14\.1.*supported GPU.*allow Hub access"): with pytest.raises(RuntimeError, match=r"kernels==0\.15\.2.*supported GPU.*allow Hub access"):
ltx2_components._create_ltx25_natten_processor() ltx2_components._create_ltx25_natten_processor()
+1 -21
View File
@@ -104,6 +104,7 @@ _OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS = frozenset(
def _stable_engine_arg_environment(monkeypatch, tmp_path): def _stable_engine_arg_environment(monkeypatch, tmp_path):
from vllm_omni import platforms from vllm_omni import platforms
monkeypatch.delenv("VLLM_USE_FLASHINFER_MOE_FP16", raising=False)
xcodec_model = tmp_path / "xcodec-model" xcodec_model = tmp_path / "xcodec-model"
xcodec_model.mkdir() xcodec_model.mkdir()
monkeypatch.setenv("XCODEC1_PATH", str(xcodec_model)) monkeypatch.setenv("XCODEC1_PATH", str(xcodec_model))
@@ -136,25 +137,6 @@ def _stable_engine_arg_environment(monkeypatch, tmp_path):
monkeypatch.setitem(sys.modules, worker_module.__name__, worker_module) monkeypatch.setitem(sys.modules, worker_module.__name__, worker_module)
def test_qwen3_omni_defaults_to_triton_moe_backend():
engine_args = {"model_arch": "Qwen3OmniMoeForConditionalGeneration"}
stage_init_utils._maybe_set_qwen3_omni_moe_backend(engine_args)
assert engine_args["moe_backend"] == "triton"
def test_qwen3_omni_preserves_explicit_moe_backend():
engine_args = {
"model_arch": "Qwen3OmniMoeForConditionalGeneration",
"moe_backend": "flashinfer",
}
stage_init_utils._maybe_set_qwen3_omni_moe_backend(engine_args)
assert engine_args["moe_backend"] == "flashinfer"
def _engine_arg_inputs(tmp_path: Path) -> tuple[PipelineConfig, DeployConfig, str]: def _engine_arg_inputs(tmp_path: Path) -> tuple[PipelineConfig, DeployConfig, str]:
parent_model = tmp_path / "parent-model" parent_model = tmp_path / "parent-model"
stage_model = tmp_path / "stage-model" stage_model = tmp_path / "stage-model"
@@ -535,8 +517,6 @@ def test_typed_engine_args_preserve_explicit_backend_default_overrides(tmp_path)
assert {name: legacy_args[name] for name in expected} == expected assert {name: legacy_args[name] for name in expected} == expected
assert {name: typed_args[name] for name in expected} == expected assert {name: typed_args[name] for name in expected} == expected
assert legacy_args["moe_backend"] == "triton"
assert typed_args["moe_backend"] == "triton"
@pytest.mark.parametrize("stage_scoped", [False, True], ids=["global", "stage-scoped"]) @pytest.mark.parametrize("stage_scoped", [False, True], ids=["global", "stage-scoped"])
@@ -203,6 +203,7 @@ _MODEL_DEPRECATE_REMOVE = (
"HIGGS_AUDIO_V2_TOKENIZER_PATH", "HIGGS_AUDIO_V2_TOKENIZER_PATH",
"VLLM_OMNI_LINGBOT_ACTION_ROOT", "VLLM_OMNI_LINGBOT_ACTION_ROOT",
"VLLM_OMNI_VOXCPM_CODE_PATH", "VLLM_OMNI_VOXCPM_CODE_PATH",
"VLLM_USE_FLASHINFER_MOE_FP16",
"XCODEC1_PATH", "XCODEC1_PATH",
"model_stage", "model_stage",
) )
-2
View File
@@ -27,7 +27,6 @@ stages:
max_num_batched_tokens: 32768 max_num_batched_tokens: 32768
max_num_seqs: 64 max_num_seqs: 64
gpu_memory_utilization: 0.9 gpu_memory_utilization: 0.9
moe_backend: triton
trust_remote_code: true trust_remote_code: true
enable_prefix_caching: false enable_prefix_caching: false
devices: "0" devices: "0"
@@ -39,7 +38,6 @@ stages:
max_num_batched_tokens: 32768 max_num_batched_tokens: 32768
max_num_seqs: 64 max_num_seqs: 64
gpu_memory_utilization: 0.6 gpu_memory_utilization: 0.6
moe_backend: triton
trust_remote_code: true trust_remote_code: true
enable_prefix_caching: false enable_prefix_caching: false
devices: "1" devices: "1"
@@ -298,7 +298,7 @@ def _create_ltx25_natten_processor() -> LTX2VideoVaeNeighborhoodNattenProcessor:
except (ImportError, OSError, RuntimeError, ValueError) as exc: except (ImportError, OSError, RuntimeError, ValueError) as exc:
raise RuntimeError( raise RuntimeError(
"LTX-2.5 DiffVAE requires the shi-labs/natten Hub kernel. " "LTX-2.5 DiffVAE requires the shi-labs/natten Hub kernel. "
"Install kernels==0.14.1, use a supported GPU, leave " "Install kernels==0.15.2, use a supported GPU, leave "
"DIFFUSERS_DISABLE_REMOTE_CODE unset, and allow Hub access during kernel initialization." "DIFFUSERS_DISABLE_REMOTE_CODE unset, and allow Hub access during kernel initialization."
) from exc ) from exc
+7 -7
View File
@@ -763,14 +763,13 @@ def prepare_engine_environment() -> None:
pass pass
def _maybe_set_qwen3_omni_moe_backend(engine_args_dict: dict[str, Any]) -> None: def _maybe_set_qwen3_omni_moe_env(engine_args_dict: dict[str, Any]) -> None:
"""Choose the stable MoE backend when Qwen3-Omni has no explicit choice."""
if ( if (
engine_args_dict.get("model_arch") == "Qwen3OmniMoeForConditionalGeneration" engine_args_dict.get("model_arch") == "Qwen3OmniMoeForConditionalGeneration"
and engine_args_dict.get("moe_backend", "auto") == "auto" and "VLLM_USE_FLASHINFER_MOE_FP16" not in os.environ
): ):
engine_args_dict["moe_backend"] = "triton" os.environ["VLLM_USE_FLASHINFER_MOE_FP16"] = "0"
logger.info("[stage_init] Set moe_backend=triton for Qwen3-Omni stage") logger.info("[stage_init] Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni stage")
def split_devices_for_replicas( def split_devices_for_replicas(
@@ -1281,8 +1280,9 @@ def _finalize_engine_args_dict(
engine_args_dict["has_sampling_extra_args"] = has_sampling_extra_args engine_args_dict["has_sampling_extra_args"] = has_sampling_extra_args
engine_args_dict["sampling_extra_args_keys"] = sampling_extra_args_keys engine_args_dict["sampling_extra_args_keys"] = sampling_extra_args_keys
# Select the typed backend option during stage argument finalization. # TODO: Remove this after the performance regression is fixed
_maybe_set_qwen3_omni_moe_backend(engine_args_dict) # Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni to avoid performance regression
_maybe_set_qwen3_omni_moe_env(engine_args_dict)
return engine_args_dict return engine_args_dict