[FEAT]Add FastVideo VSA backend for Wan2.2 (#4820)
Signed-off-by: bjf-frz <frz123db@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
3d035bfa19
commit
072bfc02dd
28 files changed
+1316
-107
No files matched your search
@@ -1,9 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field, fields
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
|
||||
from vllm.logger import init_logger
|
||||
@@ -298,24 +302,26 @@ class OmniEngineArgs(EngineArgs):
|
||||
}
|
||||
stage_connector_config["extra"]["stage_id"] = self.stage_id
|
||||
|
||||
hf_overrides = cast(dict[str, Any] | Callable[[Any], Any] | None, getattr(self, "hf_overrides", None))
|
||||
# If model_arch is specified, inject it into hf_overrides so vLLM can
|
||||
# resolve the architecture even when config.json lacks 'architectures'.
|
||||
# Also inject model_type so AutoConfig can resolve the correct config
|
||||
# class for models with empty or missing config.json (e.g. CosyVoice3).
|
||||
if self.model_arch:
|
||||
if self.hf_overrides is None:
|
||||
self.hf_overrides = {}
|
||||
if isinstance(self.hf_overrides, dict):
|
||||
self.hf_overrides.setdefault("architectures", [self.model_arch])
|
||||
if "model_type" not in self.hf_overrides:
|
||||
if hf_overrides is None:
|
||||
hf_overrides = {}
|
||||
self.hf_overrides = hf_overrides
|
||||
if isinstance(hf_overrides, dict):
|
||||
hf_overrides.setdefault("architectures", [self.model_arch])
|
||||
if "model_type" not in hf_overrides:
|
||||
model_type = _ARCH_TO_MODEL_TYPE.get(self.model_arch)
|
||||
if model_type is not None:
|
||||
self.hf_overrides.setdefault("model_type", model_type)
|
||||
hf_overrides.setdefault("model_type", model_type)
|
||||
|
||||
# Stage wrappers (e.g. Code2Wav) may need max_model_len larger
|
||||
# than the base checkpoint's text max_position_embeddings.
|
||||
if self.model_arch == "Qwen3TTSCode2Wav" and self.max_model_len is not None:
|
||||
self.hf_overrides.setdefault("talker_config", {}).setdefault(
|
||||
hf_overrides.setdefault("talker_config", {}).setdefault(
|
||||
"max_position_embeddings", int(self.max_model_len)
|
||||
)
|
||||
|
||||
@@ -331,33 +337,37 @@ class OmniEngineArgs(EngineArgs):
|
||||
if model_type is not None:
|
||||
self._patch_empty_hf_config(model_type)
|
||||
|
||||
tokenizer = cast(str | None, getattr(self, "tokenizer", None))
|
||||
|
||||
# NemotronVoiceChat: the checkpoint's only tokenizer subfolder is the
|
||||
# auxiliary rnnt_tokenizer/ (ASR loss vocab), which the generic
|
||||
# subfolder auto-detect below would wrongly pick up. The text tokenizer
|
||||
# is the Nemotron backbone's, resolved from a local override or its
|
||||
# HF id (cache-friendly).
|
||||
if not self.tokenizer and self.model_arch in (
|
||||
if not tokenizer and self.model_arch in (
|
||||
"NemotronVoiceChatThinkerForConditionalGeneration",
|
||||
"NemotronVoiceChatTalkerForConditionalGeneration",
|
||||
"NemotronVoiceChatCode2Wav",
|
||||
):
|
||||
self.tokenizer = os.environ.get("NEMOTRON_VOICECHAT_LLM_PATH") or "nvidia/NVIDIA-Nemotron-Nano-9B-v2"
|
||||
logger.info("NemotronVoiceChat: using text tokenizer from %s", self.tokenizer)
|
||||
tokenizer = self.tokenizer
|
||||
logger.info("NemotronVoiceChat: using text tokenizer from %s", tokenizer)
|
||||
|
||||
# Auto-detect tokenizer for models that store it in a subdirectory
|
||||
# rather than the root (e.g. CosyVoice3 uses CosyVoice-BlankEN/).
|
||||
if not self.tokenizer and self.model:
|
||||
if not tokenizer and self.model:
|
||||
model_path = self.model
|
||||
if os.path.isdir(model_path) and not os.path.isfile(os.path.join(model_path, "tokenizer_config.json")):
|
||||
for subfolder in sorted(os.listdir(model_path)):
|
||||
candidate = os.path.join(model_path, subfolder)
|
||||
if os.path.isdir(candidate) and os.path.isfile(os.path.join(candidate, "tokenizer_config.json")):
|
||||
self.tokenizer = candidate
|
||||
tokenizer = candidate
|
||||
logger.info("Auto-detected tokenizer at %s", candidate)
|
||||
break
|
||||
elif not os.path.isdir(model_path):
|
||||
subfolder = _TOKENIZER_SUBFOLDER_MAP.get(self.model_arch)
|
||||
if subfolder:
|
||||
tokenizer_subfolder = _TOKENIZER_SUBFOLDER_MAP.get(self.model_arch or "")
|
||||
if tokenizer_subfolder:
|
||||
# Download just the tokenizer files from the subfolder
|
||||
try:
|
||||
from huggingface_hub import snapshot_download
|
||||
@@ -365,15 +375,15 @@ class OmniEngineArgs(EngineArgs):
|
||||
local_dir = snapshot_download(
|
||||
model_path,
|
||||
allow_patterns=[
|
||||
f"{subfolder}/tokenizer*",
|
||||
f"{subfolder}/tokenization_*",
|
||||
f"{subfolder}/special_tokens*",
|
||||
f"{subfolder}/vocab*",
|
||||
f"{subfolder}/merges*",
|
||||
f"{subfolder}/added_tokens*",
|
||||
f"{tokenizer_subfolder}/tokenizer*",
|
||||
f"{tokenizer_subfolder}/tokenization_*",
|
||||
f"{tokenizer_subfolder}/special_tokens*",
|
||||
f"{tokenizer_subfolder}/vocab*",
|
||||
f"{tokenizer_subfolder}/merges*",
|
||||
f"{tokenizer_subfolder}/added_tokens*",
|
||||
],
|
||||
)
|
||||
candidate = os.path.join(local_dir, subfolder)
|
||||
candidate = os.path.join(local_dir, tokenizer_subfolder)
|
||||
if os.path.isdir(candidate):
|
||||
self.tokenizer = candidate
|
||||
logger.info("Downloaded tokenizer from %s/%s", model_path, subfolder)
|
||||
@@ -532,6 +542,7 @@ class OrchestratorArgs:
|
||||
hsdp_shard_size: int = -1
|
||||
hsdp_replicate_size: int = 1
|
||||
diffusion_attention_backend: str | None = None
|
||||
fastvideo_vsa_topk: int | None = None
|
||||
diffusion_attention_config: str | None = None
|
||||
diffusion_compile_granularity: str | None = None
|
||||
diffusion_compile_dynamic: bool | None = None
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""
|
||||
Async Omni Engine for vLLM-Omni multi-stage runtime.
|
||||
|
||||
@@ -375,9 +378,13 @@ class AsyncOmniEngine:
|
||||
getattr(self, "_duplex_runtime_extension_path", None)
|
||||
)
|
||||
if duplex_runtime_extension is not None:
|
||||
stage_clients = []
|
||||
for pool in self.stage_pools:
|
||||
assert pool.stage_client is not None
|
||||
stage_clients.append(pool.stage_client)
|
||||
validate_duplex_runtime_extension(
|
||||
duplex_runtime_extension,
|
||||
sampling_defaults=tuple(pool.stage_client.default_sampling_params for pool in self.stage_pools),
|
||||
sampling_defaults=tuple(client.default_sampling_params for client in stage_clients),
|
||||
)
|
||||
|
||||
orchestrator = Orchestrator(
|
||||
@@ -685,6 +692,7 @@ class AsyncOmniEngine:
|
||||
)
|
||||
|
||||
# Full input processing (tokenization, multimodal, etc.)
|
||||
assert self.input_processor is not None
|
||||
_t_preprocess = time.perf_counter()
|
||||
try:
|
||||
request = self.input_processor.process_inputs(
|
||||
@@ -763,11 +771,13 @@ class AsyncOmniEngine:
|
||||
Exception: Whatever prompt expansion or input processing raised.
|
||||
The caller is expected to let it reach the client.
|
||||
"""
|
||||
assert self.prompt_expand_func is not None
|
||||
expanded = self.prompt_expand_func(original_prompt, stage0_params)
|
||||
if not expanded:
|
||||
return []
|
||||
|
||||
companions: list[AddCompanionRequestMessage] = []
|
||||
assert self.input_processor is not None
|
||||
for ep in expanded:
|
||||
cid = f"{parent_id}{ep.request_id_suffix}"
|
||||
companion_prompt = ep.prompt
|
||||
@@ -864,7 +874,7 @@ class AsyncOmniEngine:
|
||||
"mag_max_skip_steps": 5,
|
||||
"mag_retention_ratio": 0.1,
|
||||
}
|
||||
if cache_backend in ("step_cache"):
|
||||
if cache_backend == "step_cache":
|
||||
return {
|
||||
"step_cache_dit_enabled": True,
|
||||
"velocity_sim_thresholds": [0.95, 0.93],
|
||||
@@ -1018,10 +1028,12 @@ class AsyncOmniEngine:
|
||||
if (
|
||||
kwargs.get("diffusion_attention_config") is not None
|
||||
or kwargs.get("diffusion_attention_backend") is not None
|
||||
or kwargs.get("fastvideo_vsa_topk") is not None
|
||||
):
|
||||
attention_config = parse_attention_config(
|
||||
kwargs.get("diffusion_attention_config"),
|
||||
attention_backend=kwargs.get("diffusion_attention_backend"),
|
||||
fastvideo_vsa_topk=kwargs.get("fastvideo_vsa_topk"),
|
||||
)
|
||||
|
||||
stage_engine_args = {
|
||||
@@ -1223,6 +1235,7 @@ class AsyncOmniEngine:
|
||||
if (
|
||||
kwargs.get("diffusion_attention_config") is not None
|
||||
or kwargs.get("diffusion_attention_backend") is not None
|
||||
or kwargs.get("fastvideo_vsa_topk") is not None
|
||||
):
|
||||
has_stage_attention = (
|
||||
hasattr(cfg.engine_args, "diffusion_attention_config")
|
||||
@@ -1232,6 +1245,7 @@ class AsyncOmniEngine:
|
||||
cfg.engine_args.diffusion_attention_config = parse_attention_config(
|
||||
kwargs.get("diffusion_attention_config"),
|
||||
attention_backend=kwargs.get("diffusion_attention_backend"),
|
||||
fastvideo_vsa_topk=kwargs.get("fastvideo_vsa_topk"),
|
||||
)
|
||||
quantization_config = kwargs.get("diffusion_quantization_config") or kwargs.get("quantization_config")
|
||||
if quantization_config is not None:
|
||||
|
||||
Reference in new issue
Block a user