[FEAT]Add FastVideo VSA backend for Wan2.2 (#4820)

Signed-off-by: bjf-frz <frz123db@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
bjf-frzandHongsheng Liu authored and GitHub committed 2026-08-24 14:48:31 +08:00
1 parent 3d035bfa19
commit 072bfc02dd
28 files changed
+1316 -107

No files matched your search

+31 -20
View File
@@ -1,9 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import argparse
import json
import os
import tempfile
from collections.abc import Callable
from dataclasses import dataclass, field, fields
from typing import Any
from typing import Any, cast
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
from vllm.logger import init_logger
@@ -298,24 +302,26 @@ class OmniEngineArgs(EngineArgs):
}
stage_connector_config["extra"]["stage_id"] = self.stage_id
hf_overrides = cast(dict[str, Any] | Callable[[Any], Any] | None, getattr(self, "hf_overrides", None))
# If model_arch is specified, inject it into hf_overrides so vLLM can
# resolve the architecture even when config.json lacks 'architectures'.
# Also inject model_type so AutoConfig can resolve the correct config
# class for models with empty or missing config.json (e.g. CosyVoice3).
if self.model_arch:
if self.hf_overrides is None:
self.hf_overrides = {}
if isinstance(self.hf_overrides, dict):
self.hf_overrides.setdefault("architectures", [self.model_arch])
if "model_type" not in self.hf_overrides:
if hf_overrides is None:
hf_overrides = {}
self.hf_overrides = hf_overrides
if isinstance(hf_overrides, dict):
hf_overrides.setdefault("architectures", [self.model_arch])
if "model_type" not in hf_overrides:
model_type = _ARCH_TO_MODEL_TYPE.get(self.model_arch)
if model_type is not None:
self.hf_overrides.setdefault("model_type", model_type)
hf_overrides.setdefault("model_type", model_type)
# Stage wrappers (e.g. Code2Wav) may need max_model_len larger
# than the base checkpoint's text max_position_embeddings.
if self.model_arch == "Qwen3TTSCode2Wav" and self.max_model_len is not None:
self.hf_overrides.setdefault("talker_config", {}).setdefault(
hf_overrides.setdefault("talker_config", {}).setdefault(
"max_position_embeddings", int(self.max_model_len)
)
@@ -331,33 +337,37 @@ class OmniEngineArgs(EngineArgs):
if model_type is not None:
self._patch_empty_hf_config(model_type)
tokenizer = cast(str | None, getattr(self, "tokenizer", None))
# NemotronVoiceChat: the checkpoint's only tokenizer subfolder is the
# auxiliary rnnt_tokenizer/ (ASR loss vocab), which the generic
# subfolder auto-detect below would wrongly pick up. The text tokenizer
# is the Nemotron backbone's, resolved from a local override or its
# HF id (cache-friendly).
if not self.tokenizer and self.model_arch in (
if not tokenizer and self.model_arch in (
"NemotronVoiceChatThinkerForConditionalGeneration",
"NemotronVoiceChatTalkerForConditionalGeneration",
"NemotronVoiceChatCode2Wav",
):
self.tokenizer = os.environ.get("NEMOTRON_VOICECHAT_LLM_PATH") or "nvidia/NVIDIA-Nemotron-Nano-9B-v2"
logger.info("NemotronVoiceChat: using text tokenizer from %s", self.tokenizer)
tokenizer = self.tokenizer
logger.info("NemotronVoiceChat: using text tokenizer from %s", tokenizer)
# Auto-detect tokenizer for models that store it in a subdirectory
# rather than the root (e.g. CosyVoice3 uses CosyVoice-BlankEN/).
if not self.tokenizer and self.model:
if not tokenizer and self.model:
model_path = self.model
if os.path.isdir(model_path) and not os.path.isfile(os.path.join(model_path, "tokenizer_config.json")):
for subfolder in sorted(os.listdir(model_path)):
candidate = os.path.join(model_path, subfolder)
if os.path.isdir(candidate) and os.path.isfile(os.path.join(candidate, "tokenizer_config.json")):
self.tokenizer = candidate
tokenizer = candidate
logger.info("Auto-detected tokenizer at %s", candidate)
break
elif not os.path.isdir(model_path):
subfolder = _TOKENIZER_SUBFOLDER_MAP.get(self.model_arch)
if subfolder:
tokenizer_subfolder = _TOKENIZER_SUBFOLDER_MAP.get(self.model_arch or "")
if tokenizer_subfolder:
# Download just the tokenizer files from the subfolder
try:
from huggingface_hub import snapshot_download
@@ -365,15 +375,15 @@ class OmniEngineArgs(EngineArgs):
local_dir = snapshot_download(
model_path,
allow_patterns=[
f"{subfolder}/tokenizer*",
f"{subfolder}/tokenization_*",
f"{subfolder}/special_tokens*",
f"{subfolder}/vocab*",
f"{subfolder}/merges*",
f"{subfolder}/added_tokens*",
f"{tokenizer_subfolder}/tokenizer*",
f"{tokenizer_subfolder}/tokenization_*",
f"{tokenizer_subfolder}/special_tokens*",
f"{tokenizer_subfolder}/vocab*",
f"{tokenizer_subfolder}/merges*",
f"{tokenizer_subfolder}/added_tokens*",
],
)
candidate = os.path.join(local_dir, subfolder)
candidate = os.path.join(local_dir, tokenizer_subfolder)
if os.path.isdir(candidate):
self.tokenizer = candidate
logger.info("Downloaded tokenizer from %s/%s", model_path, subfolder)
@@ -532,6 +542,7 @@ class OrchestratorArgs:
hsdp_shard_size: int = -1
hsdp_replicate_size: int = 1
diffusion_attention_backend: str | None = None
fastvideo_vsa_topk: int | None = None
diffusion_attention_config: str | None = None
diffusion_compile_granularity: str | None = None
diffusion_compile_dynamic: bool | None = None
+16 -2
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Async Omni Engine for vLLM-Omni multi-stage runtime.
@@ -375,9 +378,13 @@ class AsyncOmniEngine:
getattr(self, "_duplex_runtime_extension_path", None)
)
if duplex_runtime_extension is not None:
stage_clients = []
for pool in self.stage_pools:
assert pool.stage_client is not None
stage_clients.append(pool.stage_client)
validate_duplex_runtime_extension(
duplex_runtime_extension,
sampling_defaults=tuple(pool.stage_client.default_sampling_params for pool in self.stage_pools),
sampling_defaults=tuple(client.default_sampling_params for client in stage_clients),
)
orchestrator = Orchestrator(
@@ -685,6 +692,7 @@ class AsyncOmniEngine:
)
# Full input processing (tokenization, multimodal, etc.)
assert self.input_processor is not None
_t_preprocess = time.perf_counter()
try:
request = self.input_processor.process_inputs(
@@ -763,11 +771,13 @@ class AsyncOmniEngine:
Exception: Whatever prompt expansion or input processing raised.
The caller is expected to let it reach the client.
"""
assert self.prompt_expand_func is not None
expanded = self.prompt_expand_func(original_prompt, stage0_params)
if not expanded:
return []
companions: list[AddCompanionRequestMessage] = []
assert self.input_processor is not None
for ep in expanded:
cid = f"{parent_id}{ep.request_id_suffix}"
companion_prompt = ep.prompt
@@ -864,7 +874,7 @@ class AsyncOmniEngine:
"mag_max_skip_steps": 5,
"mag_retention_ratio": 0.1,
}
if cache_backend in ("step_cache"):
if cache_backend == "step_cache":
return {
"step_cache_dit_enabled": True,
"velocity_sim_thresholds": [0.95, 0.93],
@@ -1018,10 +1028,12 @@ class AsyncOmniEngine:
if (
kwargs.get("diffusion_attention_config") is not None
or kwargs.get("diffusion_attention_backend") is not None
or kwargs.get("fastvideo_vsa_topk") is not None
):
attention_config = parse_attention_config(
kwargs.get("diffusion_attention_config"),
attention_backend=kwargs.get("diffusion_attention_backend"),
fastvideo_vsa_topk=kwargs.get("fastvideo_vsa_topk"),
)
stage_engine_args = {
@@ -1223,6 +1235,7 @@ class AsyncOmniEngine:
if (
kwargs.get("diffusion_attention_config") is not None
or kwargs.get("diffusion_attention_backend") is not None
or kwargs.get("fastvideo_vsa_topk") is not None
):
has_stage_attention = (
hasattr(cfg.engine_args, "diffusion_attention_config")
@@ -1232,6 +1245,7 @@ class AsyncOmniEngine:
cfg.engine_args.diffusion_attention_config = parse_attention_config(
kwargs.get("diffusion_attention_config"),
attention_backend=kwargs.get("diffusion_attention_backend"),
fastvideo_vsa_topk=kwargs.get("fastvideo_vsa_topk"),
)
quantization_config = kwargs.get("diffusion_quantization_config") or kwargs.get("quantization_config")
if quantization_config is not None: