Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0527aadf6b | ||
|
|
6b95de36a6 |
No files matched your search
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
env:
|
env:
|
||||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
|
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
||||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||||
HF_HUB_ETAG_TIMEOUT: 60
|
HF_HUB_ETAG_TIMEOUT: 60
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
|
|
||||||
env:
|
env:
|
||||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
|
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
||||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||||
HF_HUB_ETAG_TIMEOUT: 60
|
HF_HUB_ETAG_TIMEOUT: 60
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
env:
|
env:
|
||||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
|
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
||||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||||
HF_HUB_ETAG_TIMEOUT: 60
|
HF_HUB_ETAG_TIMEOUT: 60
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
env:
|
env:
|
||||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
|
VLLM_USE_FLASHINFER_MOE_FP16: 0
|
||||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||||
HF_HUB_ETAG_TIMEOUT: 60
|
HF_HUB_ETAG_TIMEOUT: 60
|
||||||
|
|
||||||
|
|||||||
@@ -410,8 +410,8 @@ Notes:
|
|||||||
- `stages[].max_num_seqs` controls stage-level batching.
|
- `stages[].max_num_seqs` controls stage-level batching.
|
||||||
- Thinker/Talker commonly use `enforce_eager: false` for CUDA Graph paths.
|
- Thinker/Talker commonly use `enforce_eager: false` for CUDA Graph paths.
|
||||||
- Code2Wav often remains eager (`enforce_eager: true`) depending on runtime behavior.
|
- Code2Wav often remains eager (`enforce_eager: true`) depending on runtime behavior.
|
||||||
- The Qwen3-Omni deploy config pins `moe_backend: triton` for the Thinker and Talker stages. Triton has been more
|
- Qwen3-Omni defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. The Triton has been more stable & faster
|
||||||
stable and faster than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases.
|
than the FlashInfer CUTLASS unquantized MoE backend on recent vLLM rebases.
|
||||||
|
|
||||||
#### 2) Enable async chunk
|
#### 2) Enable async chunk
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ implementations to reduce numerical drift and sampling divergence.
|
|||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install kernels==0.14.1
|
pip install kernels==0.15.2
|
||||||
```
|
```
|
||||||
|
|
||||||
If `kernels` is unavailable, an explicit `FLASH_ATTN_HUB` or
|
If `kernels` is unavailable, an explicit `FLASH_ATTN_HUB` or
|
||||||
|
|||||||
@@ -17,9 +17,9 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
|||||||
|
|
||||||
The default deployment configuration situated at `vllm_omni/deploy/qwen3_omni_moe.yaml` is resolved and loaded
|
The default deployment configuration situated at `vllm_omni/deploy/qwen3_omni_moe.yaml` is resolved and loaded
|
||||||
automatically via the model registry, obviating the necessity for the `--deploy-config` flag in standard deployment topologies.
|
automatically via the model registry, obviating the necessity for the `--deploy-config` flag in standard deployment topologies.
|
||||||
The bundled Qwen3-Omni setup explicitly sets `moe_backend: triton` for the Thinker and Talker stages. This keeps
|
The bundled Qwen3-Omni setup defaults `VLLM_USE_FLASHINFER_MOE_FP16=0`. This keeps the Thinker & Talker on vLLM's
|
||||||
both stages on vLLM's Triton unquantized MoE path and avoids the unstable FlashInfer CUTLASS unquantized MoE path on
|
Triton unquantized MoE path and avoids the performance regression observed with the FlashInfer CUTLASS unquantized MoE
|
||||||
some GPUs.
|
backend.
|
||||||
Asynchronous chunk streaming is **enabled by default** within the bundled configuration.
|
Asynchronous chunk streaming is **enabled by default** within the bundled configuration.
|
||||||
|
|
||||||
To explicitly utilize a custom deployment YAML, specify the configuration path:
|
To explicitly utilize a custom deployment YAML, specify the configuration path:
|
||||||
@@ -87,9 +87,15 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
|||||||
--stage-overrides '{"1": {"gpu_memory_utilization": 0.5}}'
|
--stage-overrides '{"1": {"gpu_memory_utilization": 0.5}}'
|
||||||
```
|
```
|
||||||
|
|
||||||
To experiment with another supported MoE backend, set `moe_backend` in the
|
To experiment with the FlashInfer FP16 MoE path, set `VLLM_USE_FLASHINFER_MOE_FP16=1` before launching the server:
|
||||||
deployment YAML for both the Thinker and Talker stages. For the stage-based
|
```bash
|
||||||
CLI, pass `--moe-backend` directly on each stage command.
|
VLLM_USE_FLASHINFER_MOE_FP16=1 \
|
||||||
|
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||||
|
```
|
||||||
|
|
||||||
|
For the stage-based CLI, you usually do **not** need `--stage-overrides` for
|
||||||
|
that kind of change. Since each command launches one stage, just pass the knob
|
||||||
|
directly on that stage command:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
|
CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
|
||||||
|
|||||||
@@ -13,8 +13,9 @@ transformers >= 5.10.1, < 5.15
|
|||||||
av>=14.0.0
|
av>=14.0.0
|
||||||
omegaconf>=2.3.0
|
omegaconf>=2.3.0
|
||||||
diffusers==0.40.0
|
diffusers==0.40.0
|
||||||
# Hub kernels, including LTX-2.5 NATTEN
|
# Hub kernels, including LTX-2.5 NATTEN. Keep this pin in sync with the
|
||||||
kernels==0.14.1
|
# supported transformers range above.
|
||||||
|
kernels==0.15.2
|
||||||
safetensors>=0.8.0
|
safetensors>=0.8.0
|
||||||
accelerate==1.12.0
|
accelerate==1.12.0
|
||||||
soundfile>=0.13.1
|
soundfile>=0.13.1
|
||||||
|
|||||||
@@ -167,7 +167,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
|||||||
EnvironmentVariableCategory.PUBLIC_OMNI: 24,
|
EnvironmentVariableCategory.PUBLIC_OMNI: 24,
|
||||||
EnvironmentVariableCategory.INHERITED_VLLM: 20,
|
EnvironmentVariableCategory.INHERITED_VLLM: 20,
|
||||||
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
|
EnvironmentVariableCategory.PLATFORM_EXTERNAL: 27,
|
||||||
EnvironmentVariableCategory.MODEL_SPECIFIC: 57,
|
EnvironmentVariableCategory.MODEL_SPECIFIC: 58,
|
||||||
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
|
EnvironmentVariableCategory.BENCHMARK_TRANSITIONAL: 20,
|
||||||
EnvironmentVariableCategory.INTERNAL: 2,
|
EnvironmentVariableCategory.INTERNAL: 2,
|
||||||
}
|
}
|
||||||
@@ -182,7 +182,7 @@ def test_inventory_matches_reviewed_snapshot_counts():
|
|||||||
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 6,
|
ModelEnvironmentVariableDisposition.REQUEST_SCOPE: 6,
|
||||||
ModelEnvironmentVariableDisposition.EXTERNAL: 0,
|
ModelEnvironmentVariableDisposition.EXTERNAL: 0,
|
||||||
ModelEnvironmentVariableDisposition.INTERNALIZE: 11,
|
ModelEnvironmentVariableDisposition.INTERNALIZE: 11,
|
||||||
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 5,
|
ModelEnvironmentVariableDisposition.DEPRECATE_REMOVE: 6,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -386,7 +386,7 @@ def test_ltx25_natten_failure_has_actionable_omni_remedy(monkeypatch):
|
|||||||
MissingNattenProcessor,
|
MissingNattenProcessor,
|
||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(RuntimeError, match=r"kernels==0\.14\.1.*supported GPU.*allow Hub access"):
|
with pytest.raises(RuntimeError, match=r"kernels==0\.15\.2.*supported GPU.*allow Hub access"):
|
||||||
ltx2_components._create_ltx25_natten_processor()
|
ltx2_components._create_ltx25_natten_processor()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -104,6 +104,7 @@ _OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS = frozenset(
|
|||||||
def _stable_engine_arg_environment(monkeypatch, tmp_path):
|
def _stable_engine_arg_environment(monkeypatch, tmp_path):
|
||||||
from vllm_omni import platforms
|
from vllm_omni import platforms
|
||||||
|
|
||||||
|
monkeypatch.delenv("VLLM_USE_FLASHINFER_MOE_FP16", raising=False)
|
||||||
xcodec_model = tmp_path / "xcodec-model"
|
xcodec_model = tmp_path / "xcodec-model"
|
||||||
xcodec_model.mkdir()
|
xcodec_model.mkdir()
|
||||||
monkeypatch.setenv("XCODEC1_PATH", str(xcodec_model))
|
monkeypatch.setenv("XCODEC1_PATH", str(xcodec_model))
|
||||||
@@ -136,25 +137,6 @@ def _stable_engine_arg_environment(monkeypatch, tmp_path):
|
|||||||
monkeypatch.setitem(sys.modules, worker_module.__name__, worker_module)
|
monkeypatch.setitem(sys.modules, worker_module.__name__, worker_module)
|
||||||
|
|
||||||
|
|
||||||
def test_qwen3_omni_defaults_to_triton_moe_backend():
|
|
||||||
engine_args = {"model_arch": "Qwen3OmniMoeForConditionalGeneration"}
|
|
||||||
|
|
||||||
stage_init_utils._maybe_set_qwen3_omni_moe_backend(engine_args)
|
|
||||||
|
|
||||||
assert engine_args["moe_backend"] == "triton"
|
|
||||||
|
|
||||||
|
|
||||||
def test_qwen3_omni_preserves_explicit_moe_backend():
|
|
||||||
engine_args = {
|
|
||||||
"model_arch": "Qwen3OmniMoeForConditionalGeneration",
|
|
||||||
"moe_backend": "flashinfer",
|
|
||||||
}
|
|
||||||
|
|
||||||
stage_init_utils._maybe_set_qwen3_omni_moe_backend(engine_args)
|
|
||||||
|
|
||||||
assert engine_args["moe_backend"] == "flashinfer"
|
|
||||||
|
|
||||||
|
|
||||||
def _engine_arg_inputs(tmp_path: Path) -> tuple[PipelineConfig, DeployConfig, str]:
|
def _engine_arg_inputs(tmp_path: Path) -> tuple[PipelineConfig, DeployConfig, str]:
|
||||||
parent_model = tmp_path / "parent-model"
|
parent_model = tmp_path / "parent-model"
|
||||||
stage_model = tmp_path / "stage-model"
|
stage_model = tmp_path / "stage-model"
|
||||||
@@ -535,8 +517,6 @@ def test_typed_engine_args_preserve_explicit_backend_default_overrides(tmp_path)
|
|||||||
|
|
||||||
assert {name: legacy_args[name] for name in expected} == expected
|
assert {name: legacy_args[name] for name in expected} == expected
|
||||||
assert {name: typed_args[name] for name in expected} == expected
|
assert {name: typed_args[name] for name in expected} == expected
|
||||||
assert legacy_args["moe_backend"] == "triton"
|
|
||||||
assert typed_args["moe_backend"] == "triton"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("stage_scoped", [False, True], ids=["global", "stage-scoped"])
|
@pytest.mark.parametrize("stage_scoped", [False, True], ids=["global", "stage-scoped"])
|
||||||
|
|||||||
@@ -203,6 +203,7 @@ _MODEL_DEPRECATE_REMOVE = (
|
|||||||
"HIGGS_AUDIO_V2_TOKENIZER_PATH",
|
"HIGGS_AUDIO_V2_TOKENIZER_PATH",
|
||||||
"VLLM_OMNI_LINGBOT_ACTION_ROOT",
|
"VLLM_OMNI_LINGBOT_ACTION_ROOT",
|
||||||
"VLLM_OMNI_VOXCPM_CODE_PATH",
|
"VLLM_OMNI_VOXCPM_CODE_PATH",
|
||||||
|
"VLLM_USE_FLASHINFER_MOE_FP16",
|
||||||
"XCODEC1_PATH",
|
"XCODEC1_PATH",
|
||||||
"model_stage",
|
"model_stage",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -27,7 +27,6 @@ stages:
|
|||||||
max_num_batched_tokens: 32768
|
max_num_batched_tokens: 32768
|
||||||
max_num_seqs: 64
|
max_num_seqs: 64
|
||||||
gpu_memory_utilization: 0.9
|
gpu_memory_utilization: 0.9
|
||||||
moe_backend: triton
|
|
||||||
trust_remote_code: true
|
trust_remote_code: true
|
||||||
enable_prefix_caching: false
|
enable_prefix_caching: false
|
||||||
devices: "0"
|
devices: "0"
|
||||||
@@ -39,7 +38,6 @@ stages:
|
|||||||
max_num_batched_tokens: 32768
|
max_num_batched_tokens: 32768
|
||||||
max_num_seqs: 64
|
max_num_seqs: 64
|
||||||
gpu_memory_utilization: 0.6
|
gpu_memory_utilization: 0.6
|
||||||
moe_backend: triton
|
|
||||||
trust_remote_code: true
|
trust_remote_code: true
|
||||||
enable_prefix_caching: false
|
enable_prefix_caching: false
|
||||||
devices: "1"
|
devices: "1"
|
||||||
|
|||||||
@@ -298,7 +298,7 @@ def _create_ltx25_natten_processor() -> LTX2VideoVaeNeighborhoodNattenProcessor:
|
|||||||
except (ImportError, OSError, RuntimeError, ValueError) as exc:
|
except (ImportError, OSError, RuntimeError, ValueError) as exc:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"LTX-2.5 DiffVAE requires the shi-labs/natten Hub kernel. "
|
"LTX-2.5 DiffVAE requires the shi-labs/natten Hub kernel. "
|
||||||
"Install kernels==0.14.1, use a supported GPU, leave "
|
"Install kernels==0.15.2, use a supported GPU, leave "
|
||||||
"DIFFUSERS_DISABLE_REMOTE_CODE unset, and allow Hub access during kernel initialization."
|
"DIFFUSERS_DISABLE_REMOTE_CODE unset, and allow Hub access during kernel initialization."
|
||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
|
|||||||
@@ -763,14 +763,13 @@ def prepare_engine_environment() -> None:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _maybe_set_qwen3_omni_moe_backend(engine_args_dict: dict[str, Any]) -> None:
|
def _maybe_set_qwen3_omni_moe_env(engine_args_dict: dict[str, Any]) -> None:
|
||||||
"""Choose the stable MoE backend when Qwen3-Omni has no explicit choice."""
|
|
||||||
if (
|
if (
|
||||||
engine_args_dict.get("model_arch") == "Qwen3OmniMoeForConditionalGeneration"
|
engine_args_dict.get("model_arch") == "Qwen3OmniMoeForConditionalGeneration"
|
||||||
and engine_args_dict.get("moe_backend", "auto") == "auto"
|
and "VLLM_USE_FLASHINFER_MOE_FP16" not in os.environ
|
||||||
):
|
):
|
||||||
engine_args_dict["moe_backend"] = "triton"
|
os.environ["VLLM_USE_FLASHINFER_MOE_FP16"] = "0"
|
||||||
logger.info("[stage_init] Set moe_backend=triton for Qwen3-Omni stage")
|
logger.info("[stage_init] Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni stage")
|
||||||
|
|
||||||
|
|
||||||
def split_devices_for_replicas(
|
def split_devices_for_replicas(
|
||||||
@@ -1281,8 +1280,9 @@ def _finalize_engine_args_dict(
|
|||||||
engine_args_dict["has_sampling_extra_args"] = has_sampling_extra_args
|
engine_args_dict["has_sampling_extra_args"] = has_sampling_extra_args
|
||||||
engine_args_dict["sampling_extra_args_keys"] = sampling_extra_args_keys
|
engine_args_dict["sampling_extra_args_keys"] = sampling_extra_args_keys
|
||||||
|
|
||||||
# Select the typed backend option during stage argument finalization.
|
# TODO: Remove this after the performance regression is fixed
|
||||||
_maybe_set_qwen3_omni_moe_backend(engine_args_dict)
|
# Set VLLM_USE_FLASHINFER_MOE_FP16=0 for Qwen3-Omni to avoid performance regression
|
||||||
|
_maybe_set_qwen3_omni_moe_env(engine_args_dict)
|
||||||
return engine_args_dict
|
return engine_args_dict
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user